From 2535184dd7cf582f31cccde9cf135835d2706273 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:43:46 +0000 Subject: [PATCH 1/5] refactor: move planner scope resolution and the cost context below the plan layer The plan layer moves into its own crate next. Two pieces it uses sat in sqlbuild-python although their layer is lower: the planner scope resolution (project layer) and the cost attribution context (front layer). They now live in sqlbuild-python-project and sqlbuild-python-front, and their callers import them from there. --- .../src/cost_context/_helpers/context.rs | 48 ++++++++++ .../src/cost_context/_helpers/mod.rs | 1 + .../src/cost_context/constants.rs | 6 ++ .../src/cost_context/main/cost_aware.rs | 15 ++++ .../src/cost_context/main/instant_micros.rs | 25 ++++++ .../src/cost_context/main/mod.rs | 6 ++ .../src/cost_context/main/native_decimal.rs | 24 +++++ .../src/cost_context/main/process_object.rs | 14 +++ .../src/cost_context/main/python_decimal.rs | 21 +++++ .../main/register_cost_context.rs | 11 +++ .../src/cost_context/mod.rs | 6 ++ .../src/cost_context/models.rs} | 68 +++----------- crates/sqlbuild-python-front/src/lib.rs | 1 + .../src/planner_scope/_helpers/retained.rs | 38 ++++++++ .../src/planner_scope/main/mod.rs | 3 +- .../src/planner_scope/main/project_scope.rs | 42 --------- .../main/resolve_planner_scopes.rs} | 9 +- .../main/resolve_planner_scopes_function.rs | 10 +++ .../src/planner_scope/models.rs | 43 +++++++++ .../_helpers/audit_command/command.rs | 2 +- .../_helpers/build_command/execution.rs | 2 +- .../compile_driver/selection_phases.rs | 2 +- .../bindings/_helpers/cost_command/refresh.rs | 2 +- .../_helpers/cost_report/degradation.rs | 2 +- .../_helpers/cost_telemetry/allocation.rs | 6 +- .../_helpers/cost_telemetry/health.rs | 2 +- .../_helpers/cost_telemetry/ledger.rs | 2 +- .../bindings/_helpers/cost_telemetry/mod.rs | 2 - .../_helpers/cost_telemetry/registration.rs | 5 +- .../_helpers/cost_telemetry/values.rs | 90 ------------------- .../_helpers/freshness/publication.rs | 2 +- .../_helpers/planner_entries/command_plans.rs | 2 +- .../planner_entries/execution_plan.rs | 4 +- .../_helpers/planner_phases/buildability.rs | 2 +- .../_helpers/planner_phases/contexts.rs | 37 +------- .../_helpers/planner_phases/getters.rs | 3 +- .../_helpers/planner_phases/identities.rs | 2 +- .../bindings/_helpers/planner_phases/mod.rs | 1 - .../_helpers/planner_phases/models.rs | 9 -- .../planner_phases/output_assembly.rs | 2 +- .../_helpers/planner_phases/registration.rs | 9 +- .../planner_phases/warehouse_state.rs | 2 +- fensu.toml | 15 ++-- 43 files changed, 324 insertions(+), 274 deletions(-) create mode 100644 crates/sqlbuild-python-front/src/cost_context/_helpers/context.rs create mode 100644 crates/sqlbuild-python-front/src/cost_context/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-front/src/cost_context/constants.rs create mode 100644 crates/sqlbuild-python-front/src/cost_context/main/cost_aware.rs create mode 100644 crates/sqlbuild-python-front/src/cost_context/main/instant_micros.rs create mode 100644 crates/sqlbuild-python-front/src/cost_context/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/cost_context/main/native_decimal.rs create mode 100644 crates/sqlbuild-python-front/src/cost_context/main/process_object.rs create mode 100644 crates/sqlbuild-python-front/src/cost_context/main/python_decimal.rs create mode 100644 crates/sqlbuild-python-front/src/cost_context/main/register_cost_context.rs create mode 100644 crates/sqlbuild-python-front/src/cost_context/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs => sqlbuild-python-front/src/cost_context/models.rs} (72%) delete mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs => sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes.rs} (89%) create mode 100644 crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes_function.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs diff --git a/crates/sqlbuild-python-front/src/cost_context/_helpers/context.rs b/crates/sqlbuild-python-front/src/cost_context/_helpers/context.rs new file mode 100644 index 0000000000..561f4bb9b0 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/_helpers/context.rs @@ -0,0 +1,48 @@ +//! The `ContextVar` that holds the current cost resource context, and the scopes built over it. + +use crate::cost_context::main::process_object::process_object; +use crate::cost_context::models::CostScope; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyDict; + +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +const CONTEXT_VARIABLE_NAME: &str = "sqlbuild_cost_resource_context"; +pub(crate) const DEFAULT_PHASE: &str = "execute"; +pub(crate) const DEFAULT_ATTEMPT: i64 = 1; + +static CURRENT: PyOnceLock> = PyOnceLock::new(); + +/// The `ContextVar` holding the current cost resource context. +pub(crate) fn current_variable(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + process_object(&CURRENT, py, || { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("default", py.None())?; + imported(py, "contextvars", "ContextVar")?.call((CONTEXT_VARIABLE_NAME,), Some(&options)) + }) +} + +pub(crate) fn or_current<'py>( + value: Bound<'py, PyAny>, + current: &Bound<'py, PyAny>, + field: &str, +) -> PyResult> { + if !value.is_none() || current.is_none() { + return Ok(value); + } + native_field(current, field) +} + +pub(crate) fn scope_of(context: Bound<'_, PyAny>) -> CostScope { + CostScope { + context: Some(context.unbind()), + token: None, + } +} + +/// The active `CostResourceContext`, or `None`. +pub(crate) fn current_context(py: Python<'_>) -> PyResult> { + current_variable(py)?.call_method0("get") +} diff --git a/crates/sqlbuild-python-front/src/cost_context/_helpers/mod.rs b/crates/sqlbuild-python-front/src/cost_context/_helpers/mod.rs new file mode 100644 index 0000000000..e3c01fd8bd --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod context; diff --git a/crates/sqlbuild-python-front/src/cost_context/constants.rs b/crates/sqlbuild-python-front/src/cost_context/constants.rs new file mode 100644 index 0000000000..aca88d3b23 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/constants.rs @@ -0,0 +1,6 @@ +//! Constants of cost context. + +/// The methods an adapter with warehouse cost telemetry implements. +pub(crate) const COST_AWARE_METHODS: [&str; 2] = ["cost_capability", "collect_run_cost"]; + +pub(crate) const DECIMAL_MODULE: &str = "decimal"; diff --git a/crates/sqlbuild-python-front/src/cost_context/main/cost_aware.rs b/crates/sqlbuild-python-front/src/cost_context/main/cost_aware.rs new file mode 100644 index 0000000000..9a78915d42 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/main/cost_aware.rs @@ -0,0 +1,15 @@ +//! Whether an adapter implements the cost telemetry methods. + +use crate::cost_context::constants::COST_AWARE_METHODS; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Whether `adapter` implements the cost telemetry methods. +pub fn cost_aware(adapter: &Bound<'_, PyAny>) -> PyResult { + for name in COST_AWARE_METHODS { + if !adapter.hasattr(name)? || native_field(adapter, name)?.is_none() { + return Ok(false); + } + } + Ok(true) +} diff --git a/crates/sqlbuild-python-front/src/cost_context/main/instant_micros.rs b/crates/sqlbuild-python-front/src/cost_context/main/instant_micros.rs new file mode 100644 index 0000000000..215d1ec7fb --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/main/instant_micros.rs @@ -0,0 +1,25 @@ +//! Microseconds since the epoch: UTC for aware values, the wall clock for naive ones. + +use crate::cursor::constants::DATETIME_MODULE; +use crate::state_rows::main::event_rows::offset_aware::offset_aware; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::main::imported::imported; + +/// Microseconds since the epoch: UTC for aware values, the wall clock for naive ones. +pub fn instant_micros(value: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = value.py(); + let datetime: Bound<'_, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; + let epoch: Bound<'_, PyAny> = if offset_aware(value)? { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("tzinfo", imported(py, DATETIME_MODULE, "UTC")?)?; + datetime.call((1970, 1, 1), Some(&options))? + } else { + datetime.call1((1970, 1, 1))? + }; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("microseconds", 1)?; + let micro: Bound<'_, PyAny> = + imported(py, DATETIME_MODULE, "timedelta")?.call((), Some(&options))?; + value.sub(epoch)?.floor_div(micro)?.extract() +} diff --git a/crates/sqlbuild-python-front/src/cost_context/main/mod.rs b/crates/sqlbuild-python-front/src/cost_context/main/mod.rs new file mode 100644 index 0000000000..48557d16dd --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/main/mod.rs @@ -0,0 +1,6 @@ +pub mod cost_aware; +pub mod instant_micros; +pub mod native_decimal; +pub mod process_object; +pub mod python_decimal; +pub mod register_cost_context; diff --git a/crates/sqlbuild-python-front/src/cost_context/main/native_decimal.rs b/crates/sqlbuild-python-front/src/cost_context/main/native_decimal.rs new file mode 100644 index 0000000000..9c56b9ce38 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/main/native_decimal.rs @@ -0,0 +1,24 @@ +//! A finite non-negative `Decimal` as its exact digits and exponent. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::cost_allocation::models::PyDecimal; + +/// A finite non-negative `Decimal` as its exact digits and exponent. +pub fn native_decimal(value: &Bound<'_, PyAny>) -> PyResult { + let (sign, digits, exponent): (u8, Vec, Bound<'_, PyAny>) = + value.call_method0("as_tuple")?.extract()?; + let Ok(exponent) = exponent.extract::() else { + return Err(PyValueError::new_err("cost decimals must be finite")); + }; + if sign != 0 && digits.iter().any(|digit| *digit != 0) { + return Err(PyValueError::new_err("cost decimals must not be negative")); + } + let leading: usize = digits.iter().take_while(|digit| **digit == 0).count(); + let digits: Vec = if leading == digits.len() { + vec![0] + } else { + digits[leading..].to_vec() + }; + Ok(PyDecimal { digits, exponent }) +} diff --git a/crates/sqlbuild-python-front/src/cost_context/main/process_object.rs b/crates/sqlbuild-python-front/src/cost_context/main/process_object.rs new file mode 100644 index 0000000000..22fbe5b302 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/main/process_object.rs @@ -0,0 +1,14 @@ +//! The process-wide Python object in a cell, built on first use. + +use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyResult, Python}; +use pyo3::sync::PyOnceLock; + +/// The process-wide Python object in `cell`, built by `build` on first use. +pub fn process_object<'py>( + cell: &'static PyOnceLock>, + py: Python<'py>, + build: impl FnOnce() -> PyResult>, +) -> PyResult<&'py Bound<'py, PyAny>> { + cell.get_or_try_init(py, || Ok::, PyErr>(build()?.unbind())) + .map(|object| object.bind(py)) +} diff --git a/crates/sqlbuild-python-front/src/cost_context/main/python_decimal.rs b/crates/sqlbuild-python-front/src/cost_context/main/python_decimal.rs new file mode 100644 index 0000000000..6639d742ee --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/main/python_decimal.rs @@ -0,0 +1,21 @@ +//! `Decimal((0, digits, exponent))`. + +use crate::cost_context::constants::DECIMAL_MODULE; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_core::cost_allocation::models::PyDecimal; +use sqlbuild_python_base::boundary::main::imported::imported; + +/// `Decimal((0, digits, exponent))`. +pub fn python_decimal<'py>(py: Python<'py>, value: &PyDecimal) -> PyResult> { + let parts: Bound<'py, PyTuple> = PyTuple::new( + py, + [ + 0_i64.into_bound_py_any(py)?, + PyTuple::new(py, &value.digits)?.into_any(), + value.exponent.into_bound_py_any(py)?, + ], + )?; + imported(py, DECIMAL_MODULE, "Decimal")?.call1((parts,)) +} diff --git a/crates/sqlbuild-python-front/src/cost_context/main/register_cost_context.rs b/crates/sqlbuild-python-front/src/cost_context/main/register_cost_context.rs new file mode 100644 index 0000000000..e9f465956b --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/main/register_cost_context.rs @@ -0,0 +1,11 @@ +//! Register the cost context classes. + +use crate::cost_context::models::{CostContext, CostScope}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the cost context classes. +pub fn register_cost_context(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python-front/src/cost_context/mod.rs b/crates/sqlbuild-python-front/src/cost_context/mod.rs new file mode 100644 index 0000000000..3e4f4f853a --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/mod.rs @@ -0,0 +1,6 @@ +//! The per-resource cost attribution context and the Python cost values it reads and builds. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs b/crates/sqlbuild-python-front/src/cost_context/models.rs similarity index 72% rename from crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs rename to crates/sqlbuild-python-front/src/cost_context/models.rs index 2dfcc4d08d..9c623534a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs +++ b/crates/sqlbuild-python-front/src/cost_context/models.rs @@ -1,42 +1,23 @@ -//! `CostContext`: the per-resource cost attribution context, held in a Python `ContextVar`. +//! Public records of cost context. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; +use crate::cost_context::_helpers::context::{ + DEFAULT_ATTEMPT, DEFAULT_PHASE, current_context, current_variable, or_current, scope_of, +}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple}; use pyo3::{IntoPyObjectExt, pyclass, pymethods}; use sqlbuild_host::entries::models::ExecutorHost; - -use crate::bindings::_helpers::cost_telemetry::values::process_object; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; -const CONTEXT_VARIABLE_NAME: &str = "sqlbuild_cost_resource_context"; -const DEFAULT_PHASE: &str = "execute"; -const DEFAULT_ATTEMPT: i64 = 1; - -static CURRENT: PyOnceLock> = PyOnceLock::new(); - -fn current_variable(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { - process_object(&CURRENT, py, || { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("default", py.None())?; - imported(py, "contextvars", "ContextVar")?.call((CONTEXT_VARIABLE_NAME,), Some(&options)) - }) -} - -/// The active `CostResourceContext`, or `None`. -pub(crate) fn current_context(py: Python<'_>) -> PyResult> { - current_variable(py)?.call_method0("get") -} - /// A `with` scope that makes one resource context current, or does nothing without one. #[pyclass(module = "sqlbuild._native")] -pub(crate) struct CostScope { - context: Option>, - token: Option>, +#[derive(Debug)] +pub struct CostScope { + pub(crate) context: Option>, + pub(crate) token: Option>, } #[pymethods] @@ -61,27 +42,10 @@ impl CostScope { } } -fn or_current<'py>( - value: Bound<'py, PyAny>, - current: &Bound<'py, PyAny>, - field: &str, -) -> PyResult> { - if !value.is_none() || current.is_none() { - return Ok(value); - } - native_field(current, field) -} - -fn scope_of(context: Bound<'_, PyAny>) -> CostScope { - CostScope { - context: Some(context.unbind()), - token: None, - } -} - /// Run-cost execution context entry points. #[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct CostContext; +#[derive(Debug)] +pub struct CostContext; #[pymethods] impl CostContext { @@ -92,7 +56,7 @@ impl CostContext { #[staticmethod] #[pyo3(signature = (**options))] - pub(crate) fn scope(py: Python<'_>, options: Option>) -> PyResult { + pub fn scope(py: Python<'_>, options: Option>) -> PyResult { let arguments: KeywordArguments<'_> = KeywordArguments::read( py, "scope", @@ -139,7 +103,7 @@ impl CostContext { #[staticmethod] #[pyo3(signature = (**options))] - pub(crate) fn resource_scope( + pub fn resource_scope( py: Python<'_>, options: Option>, ) -> PyResult { @@ -187,9 +151,3 @@ impl CostContext { Ok(scope_of(context)) } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/lib.rs b/crates/sqlbuild-python-front/src/lib.rs index b0070dbde8..183d051c5e 100644 --- a/crates/sqlbuild-python-front/src/lib.rs +++ b/crates/sqlbuild-python-front/src/lib.rs @@ -8,6 +8,7 @@ pub mod command_runtime; pub mod compile_attachment; pub mod compile_driver; pub mod contracts; +pub mod cost_context; pub mod cursor; pub mod discovery; pub mod macro_bridge; diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs index bd56463a03..06ac1d20c1 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs @@ -10,8 +10,11 @@ use std::sync::Arc; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use crate::native_host::models::ModelHandle; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_base::records::models::NativeRecord; +use sqlbuild_python_base::records::types::RecordId; /// Read native ownership directly, or import a caller-authored public scope at the API edge. pub(crate) fn scope_data(value: &Bound<'_, PyAny>) -> PyResult> { @@ -54,3 +57,38 @@ pub(crate) fn scope_data(value: &Bound<'_, PyAny>) -> PyResult> { .collect(), })) } + +/// Preserve public record identity and dataclass behavior on explicit projection. +pub(crate) fn project_scope<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let Ok(scope) = value.cast::() else { + return Ok(value.clone()); + }; + scope + .get() + .projection + .get_or_try_init(value.py(), || { + let names = [ + "upstream_deps", + "downstream_deps", + "all_keys", + "models_by_name", + "selected_keys", + "execution_order", + "user_selected_keys", + "python_read_source_names", + "sql_test_selection", + "load_sources", + "end_audit_source_keys", + ]; + let fields = names + .iter() + .map(|name| Ok((*name, scope.get().data.field(value.py(), name)?))) + .collect::>>()?; + Ok::<_, pyo3::PyErr>( + new_record_with(value.py(), RecordId::PlannerScope, &fields)? + .cast_into::() + .map(Bound::unbind)?, + ) + }) + .map(|value| value.bind(scope.py()).clone().into_any()) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/mod.rs b/crates/sqlbuild-python-project/src/planner_scope/main/mod.rs index 03a88c4a60..84294e6dc1 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/mod.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/mod.rs @@ -9,7 +9,8 @@ pub mod edges; pub mod expand_selected_loader_dependencies_function; pub mod keys; pub mod project_graph; -pub mod project_scope; +pub mod resolve_planner_scopes; +pub mod resolve_planner_scopes_function; pub mod scope_data; pub mod source_load_map; pub mod upstream_loader_dependency_names; diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs b/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs deleted file mode 100644 index 734d1789c4..0000000000 --- a/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs +++ /dev/null @@ -1,42 +0,0 @@ -//! Preserve public record identity and dataclass behavior on explicit projection. - -use crate::planner_scope::models::RetainedScope; -use pyo3::prelude::{Bound, PyAny, PyResult}; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::models::NativeRecord; -use sqlbuild_python_base::records::types::RecordId; - -/// Preserve public record identity and dataclass behavior on explicit projection. -pub fn project_scope<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - let Ok(scope) = value.cast::() else { - return Ok(value.clone()); - }; - scope - .get() - .projection - .get_or_try_init(value.py(), || { - let names = [ - "upstream_deps", - "downstream_deps", - "all_keys", - "models_by_name", - "selected_keys", - "execution_order", - "user_selected_keys", - "python_read_source_names", - "sql_test_selection", - "load_sources", - "end_audit_source_keys", - ]; - let fields = names - .iter() - .map(|name| Ok((*name, scope.get().data.field(value.py(), name)?))) - .collect::>>()?; - Ok::<_, pyo3::PyErr>( - new_record_with(value.py(), RecordId::PlannerScope, &fields)? - .cast_into::() - .map(Bound::unbind)?, - ) - }) - .map(|value| value.bind(scope.py()).clone().into_any()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs b/crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs rename to crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes.rs index 6e72967165..73e1b9f8b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes.rs @@ -1,18 +1,19 @@ //! Resolve the selected, stale-warning and inspection scopes of one plan. -use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; +use crate::planner_scope::main::build_planner_scope::build_planner_scope; +use crate::planner_scope::main::scope_data::scope_data; +use crate::planner_scope::models::PlannerScopeResolution; use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::pyfunction; use pyo3::types::PyTuple; use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_project::planner_scope::main::build_planner_scope::build_planner_scope; -use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; #[pyfunction] #[pyo3(signature = (*, project, selection, policies))] -pub(crate) fn resolve_planner_scopes<'py>( +/// The selected, stale-warning and inspection scopes `selection` resolves over `project`. +pub fn resolve_planner_scopes<'py>( project: &Bound<'py, PyAny>, selection: &Bound<'py, PyAny>, policies: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes_function.rs new file mode 100644 index 0000000000..3df9aa64de --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes_function.rs @@ -0,0 +1,10 @@ +//! `resolve_planner_scopes` as a function of `module`. + +use crate::planner_scope::main::resolve_planner_scopes::resolve_planner_scopes; + +/// `resolve_planner_scopes` as a function of `module`. +pub fn resolve_planner_scopes_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(resolve_planner_scopes, module) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/models.rs b/crates/sqlbuild-python-project/src/planner_scope/models.rs index 91a26242c2..e807f3c8f9 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/models.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/models.rs @@ -3,6 +3,7 @@ use crate::graph::main::project::edge_dict::edge_dict; use crate::graph::models::KeyCache; use crate::native_host::models::ModelHandle; +use crate::planner_scope::_helpers::retained::project_scope; use pyo3::IntoPyObjectExt; use pyo3::prelude::Bound; use pyo3::prelude::Py; @@ -10,6 +11,7 @@ use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; use pyo3::prelude::Python; use pyo3::pyclass; +use pyo3::pymethods; use pyo3::sync::PyOnceLock; use pyo3::types::PyDict; use pyo3::types::PyDictMethods; @@ -129,3 +131,44 @@ impl ScopeData { } } } + +/// Resolved planner scopes for one plan build. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct PlannerScopeResolution { + pub selected_scope: Py, + pub stale_warning_scope: Py, + pub inspection_scope: Py, +} + +#[pymethods] +impl PlannerScopeResolution { + #[new] + #[pyo3(signature = (*, selected_scope, stale_warning_scope, inspection_scope))] + fn new( + selected_scope: Py, + stale_warning_scope: Py, + inspection_scope: Py, + ) -> Self { + Self { + selected_scope, + stale_warning_scope, + inspection_scope, + } + } + + #[getter] + fn selected_scope<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { + project_scope(self.selected_scope.bind(py)) + } + + #[getter] + fn stale_warning_scope<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { + project_scope(self.stale_warning_scope.bind(py)) + } + + #[getter] + fn inspection_scope<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { + project_scope(self.inspection_scope.bind(py)) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 8eb7797022..05d1c80c5f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -22,7 +22,6 @@ use crate::bindings::_helpers::build_progress::progress_text::write_execution_he use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; -use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::planner_entries::command_plans::build_audit_command_plan; use crate::bindings::_helpers::result_output::documents::{ @@ -42,6 +41,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_front::cost_context::models::CostContext; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; use sqlbuild_python_project::project_snapshot::types::VerifiedRun; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 22c526cf64..ddce14bdbd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -31,7 +31,6 @@ use crate::bindings::_helpers::consumption::policy::{changed_policy, policy_line use crate::bindings::_helpers::consumption::recorder::{ ConsumptionRecorder, Prepared, held_back_text, prepare as prepare_consumption, run_reason_text, }; -use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::freshness::publication::append_freshness_records; use crate::bindings::_helpers::observations::publication::append_build_observations; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; @@ -56,6 +55,7 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cost_context::models::CostContext; use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; use sqlbuild_python_front::native_host::main::recorder_callbacks::recorder_callbacks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index 005212247d..1ec02765cb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -8,13 +8,13 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::assembly::assemble_project; use crate::bindings::_helpers::compile_driver::pipeline_entries::compile_analysis_selection; -use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_scope::main::resolve_planner_scopes::resolve_planner_scopes; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use sqlbuild_python_project::planner_scope::models::ScopeData; use sqlbuild_python_project::planner_selection::main::split_project_unit_test_selectors::split_project_unit_test_selectors; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index b19e980701..6b3a3776b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -15,7 +15,6 @@ use crate::bindings::_helpers::compile_driver::discovery::discover_project_input use crate::bindings::_helpers::cost_report::functions::read_cost_run; use crate::bindings::_helpers::cost_report::functions::resolve_cost_run; use crate::bindings::_helpers::cost_report::functions::write_cost_run; -use crate::bindings::_helpers::cost_telemetry::values::cost_aware; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -26,6 +25,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_front::compile_attachment::main::build_effective_target_config::build_effective_target_config; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::cost_context::main::cost_aware::cost_aware; /// Collection budget of an explicit `sqb cost` refresh, which a user waits for on purpose. const REFRESH_BUDGET_SECONDS: f64 = 300.0; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs index 8935152ffc..39ea17272d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs @@ -16,12 +16,12 @@ use crate::bindings::_helpers::cost_report::attribution::report_cost_status; use crate::bindings::_helpers::cost_report::publication::publish_resource_costs; use crate::bindings::_helpers::cost_report::records::cost_run; use crate::bindings::_helpers::cost_report::store; -use crate::bindings::_helpers::cost_telemetry::values::cost_aware; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cost_context::main::cost_aware::cost_aware; const LOGGER_NAME: &str = "sqlbuild.cli.commands._helpers.cost.collection"; const COLLECTION_LOGGER_NAME: &str = "sqlbuild.cost"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs index f3cd30f4c2..ff7d020b11 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs @@ -10,15 +10,15 @@ use sqlbuild_core::cost_allocation::models::{ }; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::cost_telemetry::values::{ - instant_micros, native_decimal, python_decimal, -}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cost_context::main::instant_micros::instant_micros; +use sqlbuild_python_front::cost_context::main::native_decimal::native_decimal; +use sqlbuild_python_front::cost_context::main::python_decimal::python_decimal; const PENDING_MESSAGE: &str = "Snowflake query history is not complete yet."; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs index 55ccc7814e..9539893b06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs @@ -8,7 +8,7 @@ use pyo3::sync::PyOnceLock; use pyo3::types::PyTypeMethods; use pyo3::{pyclass, pymethods}; -use crate::bindings::_helpers::cost_telemetry::values::process_object; +use sqlbuild_python_front::cost_context::main::process_object::process_object; const HEALTH_ATTRIBUTE: &str = "COST_TELEMETRY_HEALTH"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs index 0f8c8fc14c..b02639c427 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs @@ -16,7 +16,6 @@ use sqlbuild_core::json::main::dumps::dumps; use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; use crate::bindings::_helpers::cost_telemetry::health::mark_ledger_failure; -use crate::bindings::_helpers::cost_telemetry::values::process_object; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -26,6 +25,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cost_context::main::process_object::process_object; use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; static LEDGER_LOCK: PyOnceLock> = PyOnceLock::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs index 89f46b212d..de3d311170 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs @@ -1,7 +1,5 @@ pub(crate) mod allocation; -pub(crate) mod context; pub(crate) mod health; pub(crate) mod ledger; pub(crate) mod registration; pub(crate) mod snowflake; -pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs index 4ce5de8966..f9332a6bf3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs @@ -2,11 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::cost_telemetry::{allocation, context, health, ledger, snowflake}; +use crate::bindings::_helpers::cost_telemetry::{allocation, health, ledger, snowflake}; +use sqlbuild_python_front::cost_context::main::register_cost_context::register_cost_context; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { allocation::register(module)?; - context::register(module)?; + register_cost_context(module)?; health::register(module)?; ledger::register(module)?; snowflake::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs deleted file mode 100644 index 5a415eec9a..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs +++ /dev/null @@ -1,90 +0,0 @@ -//! Python `Decimal` and `datetime` values as exact native decimals and microsecond instants. - -use pyo3::IntoPyObjectExt; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyTuple}; -use sqlbuild_core::cost_allocation::models::PyDecimal; - -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; -use sqlbuild_python_front::state_rows::main::event_rows::offset_aware::offset_aware; - -const DECIMAL_MODULE: &str = "decimal"; -/// The methods an adapter with warehouse cost telemetry implements. -const COST_AWARE_METHODS: [&str; 2] = ["cost_capability", "collect_run_cost"]; - -/// Whether `adapter` implements the cost telemetry methods. -pub(crate) fn cost_aware(adapter: &Bound<'_, PyAny>) -> PyResult { - for name in COST_AWARE_METHODS { - if !adapter.hasattr(name)? || native_field(adapter, name)?.is_none() { - return Ok(false); - } - } - Ok(true) -} - -/// A finite non-negative `Decimal` as its exact digits and exponent. -pub(crate) fn native_decimal(value: &Bound<'_, PyAny>) -> PyResult { - let (sign, digits, exponent): (u8, Vec, Bound<'_, PyAny>) = - value.call_method0("as_tuple")?.extract()?; - let Ok(exponent) = exponent.extract::() else { - return Err(PyValueError::new_err("cost decimals must be finite")); - }; - if sign != 0 && digits.iter().any(|digit| *digit != 0) { - return Err(PyValueError::new_err("cost decimals must not be negative")); - } - let leading: usize = digits.iter().take_while(|digit| **digit == 0).count(); - let digits: Vec = if leading == digits.len() { - vec![0] - } else { - digits[leading..].to_vec() - }; - Ok(PyDecimal { digits, exponent }) -} - -/// `Decimal((0, digits, exponent))`. -pub(crate) fn python_decimal<'py>( - py: Python<'py>, - value: &PyDecimal, -) -> PyResult> { - let parts: Bound<'py, PyTuple> = PyTuple::new( - py, - [ - 0_i64.into_bound_py_any(py)?, - PyTuple::new(py, &value.digits)?.into_any(), - value.exponent.into_bound_py_any(py)?, - ], - )?; - imported(py, DECIMAL_MODULE, "Decimal")?.call1((parts,)) -} - -/// Microseconds since the epoch: UTC for aware values, the wall clock for naive ones. -pub(crate) fn instant_micros(value: &Bound<'_, PyAny>) -> PyResult { - let py: Python<'_> = value.py(); - let datetime: Bound<'_, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; - let epoch: Bound<'_, PyAny> = if offset_aware(value)? { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("tzinfo", imported(py, DATETIME_MODULE, "UTC")?)?; - datetime.call((1970, 1, 1), Some(&options))? - } else { - datetime.call1((1970, 1, 1))? - }; - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("microseconds", 1)?; - let micro: Bound<'_, PyAny> = - imported(py, DATETIME_MODULE, "timedelta")?.call((), Some(&options))?; - value.sub(epoch)?.floor_div(micro)?.extract() -} - -/// The process-wide Python object in `cell`, built by `build` on first use. -pub(crate) fn process_object<'py>( - cell: &'static PyOnceLock>, - py: Python<'py>, - build: impl FnOnce() -> PyResult>, -) -> PyResult<&'py Bound<'py, PyAny>> { - cell.get_or_try_init(py, || Ok::, PyErr>(build()?.unbind())) - .map(|object| object.bind(py)) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs index c7a628f3a1..0a3336c014 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs @@ -5,7 +5,6 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::freshness::state_io::write_source_freshness_records; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -18,6 +17,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cost_context::models::CostContext; /// Observed records grouped by `(database, schema)` in first-seen order. type RecordsByLocation<'py> = Vec<( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index 8e402988e3..2aa469ea8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -10,7 +10,6 @@ use crate::bindings::_helpers::planner_entries::values::{ }; use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; use crate::bindings::_helpers::planner_identity::metadata::build_seed_identity; -use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; use crate::bindings::_helpers::planner_strategy::audit_entries::build_selected_audit_entries; use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; @@ -33,6 +32,7 @@ use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_scope::main::resolve_planner_scopes::resolve_planner_scopes; use sqlbuild_python_project::resource_views::main::function_reads::deps::deps; use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index 1599f248c9..66962e8d0b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -20,7 +20,6 @@ use crate::bindings::_helpers::planner_phases::identities::{ }; use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerRuntime; -use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::models::{ PlanOutputContext, PlanWarningContext, PlannedModelEntries, StoragePolicyContext, }; @@ -30,7 +29,6 @@ use crate::bindings::_helpers::planner_phases::output_assembly::{ base_plan_output, }; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; -use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; use crate::bindings::_helpers::planner_phases::warehouse_state::gather_planner_warehouse_state; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::planner_warehouse::source_freshness::build_planner_source_freshness_result; @@ -44,7 +42,9 @@ use sqlbuild_python_base::build_command::main::utc_now::utc_now; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_scope::main::resolve_planner_scopes::resolve_planner_scopes; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use std::time::Instant; const INSPECTION_CATALOG: AdapterHost = AdapterHost::OpenInspectionCatalog; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs index 9dbc69077b..dfb5acccd7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs @@ -2,7 +2,6 @@ use crate::bindings::_helpers::planner_phases::constants::MISSING_UPSTREAM_CODE; use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_warehouse::source_tables::check_selected_source_tables_exist; @@ -16,6 +15,7 @@ use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::planner_identity::main::execution_origins::execution_origins; use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use sqlbuild_python_project::resource_views::main::seed_reads::external::external; use sqlbuild_python_project::resource_views::main::seed_reads::key::key; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs index c009111ea3..106761dc8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs @@ -4,42 +4,7 @@ use pyo3::prelude::{Bound, Py, PyAny, PyResult}; use pyo3::pymethods; use pyo3::{PyClass, pyclass::boolean_struct::True}; -use crate::bindings::_helpers::planner_phases::models::{ - PlannerIdentityContext, PlannerScopeResolution, -}; -use sqlbuild_python_project::planner_scope::main::project_scope::project_scope; - -#[pymethods] -impl PlannerScopeResolution { - #[new] - #[pyo3(signature = (*, selected_scope, stale_warning_scope, inspection_scope))] - fn new( - selected_scope: Py, - stale_warning_scope: Py, - inspection_scope: Py, - ) -> Self { - Self { - selected_scope, - stale_warning_scope, - inspection_scope, - } - } - - #[getter] - fn selected_scope<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { - project_scope(self.selected_scope.bind(py)) - } - - #[getter] - fn stale_warning_scope<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { - project_scope(self.stale_warning_scope.bind(py)) - } - - #[getter] - fn inspection_scope<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { - project_scope(self.inspection_scope.bind(py)) - } -} +use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; #[pymethods] impl PlannerIdentityContext { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs index ca87274851..8aa33875fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs @@ -4,10 +4,11 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use crate::bindings::_helpers::planner_phases::models::{ - PlannerIdentityContext, PlannerRuntime, PlannerScopeResolution, PlannerWarehouseState, + PlannerIdentityContext, PlannerRuntime, PlannerWarehouseState, }; use sqlbuild_python_base::records::main::registry::class_of::class_of; use sqlbuild_python_base::records::models::GetterEntry; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use sqlbuild_python_project::planner_scope::models::RetainedScope; /// The getter readers of this module's native pyclasses. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs index 19ae42d660..c2562c53c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -5,7 +5,6 @@ use crate::bindings::_helpers::planner_identity::identities::direct_model_versio use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::models::IdentityWork; use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; -use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::sync::PyOnceLock; @@ -21,6 +20,7 @@ use sqlbuild_python_project::compiled_views::main::project::functions::functions use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; use sqlbuild_python_project::planner_phases::main::native::native; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use std::collections::{BTreeMap, HashSet}; static IDENTITY_WORK: PyOnceLock> = PyOnceLock::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs index c952ce457a..b095876f5b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs @@ -13,5 +13,4 @@ pub(crate) mod output_assembly; pub(crate) mod registration; pub(crate) mod retention; pub(crate) mod runtime; -pub(crate) mod scopes; pub(crate) mod warehouse_state; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs index fc33a2e8b3..c1479d9f2b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs @@ -90,15 +90,6 @@ pub(crate) struct PlannerWarehouseState { pub(crate) migration_fingerprints: Py, } -/// Resolved planner scopes for one plan build. -#[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug)] -pub(crate) struct PlannerScopeResolution { - pub(crate) selected_scope: Py, - pub(crate) stale_warning_scope: Py, - pub(crate) inspection_scope: Py, -} - /// Expected version identities for inspection and stale-warning scopes. #[pyclass(module = "sqlbuild._native", frozen, get_all)] #[derive(Debug)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 26493cd00c..14c743ae18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -6,7 +6,6 @@ use crate::bindings::_helpers::planner_identity::staleness::build_stale_out_of_s use crate::bindings::_helpers::planner_migrations::fingerprint::with_migration_fingerprints; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; -use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::models::{ PlanOutputContext, PlanOutputFields, PlanWarningContext, PlannedModelEntries, StoragePolicyContext, @@ -33,6 +32,7 @@ use sqlbuild_python_project::planner_phases::main::strings::strings; use sqlbuild_python_project::planner_phases::main::text::text; use sqlbuild_python_project::planner_phases::main::tuple::tuple; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; fn concat<'py>(py: Python<'py>, parts: &[Bound<'py, PyAny>]) -> PyResult> { let mut items: Vec<_> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs index 5a57fe9c6b..5a1d86ec23 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -2,19 +2,16 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; +use sqlbuild_python_project::planner_scope::main::resolve_planner_scopes_function::resolve_planner_scopes_function; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module .add_class::()?; - module - .add_class::()?; + module.add_class::()?; module .add_class::()?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes, - module - )?)?; + module.add_function(resolve_planner_scopes_function(module)?)?; module.add_function(wrap_pyfunction!( crate::bindings::_helpers::planner_phases::identities::planner_identity_work_info, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index a682495b34..1150401e9e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -10,7 +10,6 @@ use crate::bindings::_helpers::planner_migrations::models::{ use crate::bindings::_helpers::planner_migrations::old_name_planning::plan_old_name_views as plan_old_name_migration_views; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; -use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::models::PlannerWarehouseState; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; @@ -22,6 +21,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_project::graph::main::project::compiled_graph_keys::compiled_graph_keys; use sqlbuild_python_project::planner_phases::main::native::native; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use std::time::Instant; fn progress(runtime: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { diff --git a/fensu.toml b/fensu.toml index 7243c0573f..a7413cae2b 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2154,6 +2154,11 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs=3", "crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs=1", "crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs=5", + "crates/sqlbuild-python-front/src/cost_context/_helpers/context.rs=3", + "crates/sqlbuild-python-front/src/cost_context/main/instant_micros.rs=6", + "crates/sqlbuild-python-front/src/cost_context/main/native_decimal.rs=1", + "crates/sqlbuild-python-front/src/cost_context/main/python_decimal.rs=2", + "crates/sqlbuild-python-front/src/cost_context/models.rs=2", "crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs=0", "crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs=1", "crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs=0", @@ -2437,11 +2442,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=23", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs=7", @@ -2771,6 +2774,7 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/authored_values/models.rs=6", "crates/sqlbuild-python-front/src/compile_driver/models.rs=3", "crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs=1", + "crates/sqlbuild-python-front/src/cost_context/models.rs=2", "crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs=1", "crates/sqlbuild-python-front/src/cursor/models.rs=4", "crates/sqlbuild-python-front/src/discovery/_helpers/failures/faults.rs=2", @@ -2819,6 +2823,7 @@ pending_budgets = [ "crates/sqlbuild-python-project/src/graph/models.rs=10", "crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs=2", "crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs=1", + "crates/sqlbuild-python-project/src/planner_scope/models.rs=3", "crates/sqlbuild-python-project/src/planner_selection/models.rs=1", "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs=6", "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs=3", @@ -2878,7 +2883,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs=1", @@ -2946,7 +2950,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs=66", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=41", + "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=38", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs=7", @@ -3032,8 +3036,6 @@ module_layers = [ "cost_command=commands", "cost_report=output", "cost_telemetry=output", - "cost_telemetry/context=front", - "cost_telemetry/values=front", "cursor=registry", "dag_artifact=compile", "dagster_invocation=commands", @@ -3081,7 +3083,6 @@ module_layers = [ "planner_migrations=plan", "planner_phases=plan", "planner_phases/models=project", - "planner_phases/scopes=project", "planner_registration=registry", "planner_resolution=plan", "planner_scenario=plan", From c99b3e5c9f07d654c3cd4b866b26103436155030 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 18:10:03 +0000 Subject: [PATCH 2/5] refactor: move the plan layer of the Python bindings into sqlbuild-python-planner Work in progress: the move itself. main/ forwarders are not folded yet, and the field-fallback list, duplicate allowlist and crate README are not updated. --- Cargo.lock | 26 +- Cargo.toml | 5 +- crates/sqlbuild-python-planner/Cargo.toml | 30 ++ .../src/consumption/_helpers}/facts.rs | 14 +- .../src/consumption/_helpers}/mod.rs | 0 .../src/consumption/_helpers}/policy.rs | 0 .../src/consumption/_helpers}/recorder.rs | 221 +---------- .../consumption/_helpers}/relation_reads.rs | 0 .../src/consumption/main/changed_policy.rs | 17 + .../src/consumption/main/held_back_text.rs | 10 + .../src/consumption/main/kind_value.rs | 10 + .../src/consumption/main/mod.rs | 9 + .../src/consumption/main/policy_line.rs | 10 + .../src/consumption/main/prepare.rs | 24 ++ .../src/consumption/main/preview_changed.rs | 22 ++ .../src/consumption/main/reach_value.rs | 10 + .../src/consumption/main/relation.rs | 13 + .../src/consumption/main/run_reason_text.rs | 10 + .../src/consumption/mod.rs | 5 + .../src/consumption/models.rs | 220 +++++++++++ .../freshness/_helpers}/exception_types.rs | 0 .../freshness/_helpers}/freshness_observe.rs | 16 +- .../src/freshness/_helpers/mod.rs | 7 + .../src/freshness/_helpers}/observation.rs | 20 +- .../src/freshness/_helpers}/planning.rs | 21 +- .../src/freshness/_helpers}/publication.rs | 2 +- .../src/freshness/_helpers}/state_io.rs | 18 +- .../src/freshness/_helpers}/support.rs | 5 +- .../src/freshness/constants.rs | 4 + .../main/append_freshness_records.rs | 22 ++ ...urce_freshness_planning_result_function.rs | 10 + ...e_freshness_propagation_result_function.rs | 10 + .../src/freshness/main/mod.rs | 15 + .../freshness/main/normalize_observation.rs | 13 + .../src/freshness/main/observe_freshness.rs | 19 + .../freshness/main/presumed_utc_datetime.rs | 12 + .../src/freshness/main/read_latest.rs | 18 + .../main/register_exception_types.rs | 12 + .../main/register_freshness_observe.rs | 12 + .../freshness/main/register_observation.rs | 12 + .../src/freshness/main/register_planning.rs | 12 + .../freshness/main/register_publication.rs | 12 + .../src/freshness/main/register_state_io.rs | 12 + .../main/require_aware_utc_datetime.rs | 15 + .../src/freshness/main/row_record.rs | 12 + .../src/freshness/mod.rs | 6 + .../src/freshness/models.rs | 27 ++ crates/sqlbuild-python-planner/src/lib.rs | 19 + .../src/lifecycle_events/_helpers}/catalog.rs | 0 .../lifecycle_events/_helpers}/envelopes.rs | 12 +- .../src/lifecycle_events/_helpers/mod.rs | 4 + .../lifecycle_events/_helpers}/payloads.rs | 0 .../lifecycle_events/_helpers}/validation.rs | 7 +- .../src/lifecycle_events/main/mod.rs | 3 + .../lifecycle_events/main/register_catalog.rs | 12 + .../main/register_envelopes.rs | 12 + .../main/register_validation.rs | 12 + .../src/lifecycle_events/mod.rs | 4 + .../src/microbatches/_helpers}/conversion.rs | 0 .../src/microbatches/_helpers/mod.rs | 3 + .../src/microbatches/_helpers}/projection.rs | 7 +- .../src/microbatches/_helpers}/statements.rs | 3 +- .../src/microbatches/main/mod.rs | 3 + .../microbatches/main/register_projection.rs | 12 + .../microbatches/main/register_statements.rs | 12 + .../src/microbatches/main/timestamp.rs | 12 + .../src/microbatches/mod.rs | 4 + .../src/observations/_helpers}/mod.rs | 0 .../src/observations/_helpers}/publication.rs | 9 +- .../observations/_helpers}/source_capture.rs | 0 .../main/append_build_observations.rs | 19 + .../observations/main/arm_source_capture.rs | 9 + .../main/capture_source_columns.rs | 9 + .../src/observations/main/mod.rs | 4 + .../observations/main/take_source_capture.rs | 10 + .../src/observations/mod.rs | 4 + .../plan_output/_helpers}/cursor_details.rs | 8 +- .../src/plan_output/_helpers}/display.rs | 3 +- .../src/plan_output/_helpers}/migrations.rs | 11 +- .../src/plan_output/_helpers/mod.rs | 8 + .../src/plan_output/_helpers}/plan_json.rs | 33 +- .../src/plan_output/_helpers}/plan_text.rs | 31 +- .../plan_output/_helpers}/python_identity.rs | 0 .../src/plan_output/_helpers}/safety.rs | 4 +- .../src/plan_output/_helpers}/values.rs | 0 .../src/plan_output/constants.rs | 4 + .../main/aligned_timestamp_bounds.rs | 16 + .../src/plan_output/main/any.rs | 14 + .../plan_output/main/cursor_bound_display.rs | 16 + .../src/plan_output/main/dict.rs | 17 + .../src/plan_output/main/format_plan.rs | 13 + .../src/plan_output/main/format_plan_json.rs | 13 + .../plan_output/main/future_cursor_safety.rs | 12 + .../plan_output/main/inclusive_cursor_end.rs | 16 + .../src/plan_output/main/items.rs | 12 + .../plan_output/main/maximum_start_safety.rs | 12 + .../src/plan_output/main/mod.rs | 19 + .../src/plan_output/main/optional_facts.rs | 13 + .../src/plan_output/main/register_display.rs | 12 + .../plan_output/main/register_migrations.rs | 12 + .../plan_output/main/register_plan_json.rs | 12 + .../plan_output/main/register_plan_text.rs | 12 + .../src/plan_output/main/register_safety.rs | 12 + .../main/resolve_bounded_cursor_override.rs | 22 ++ ...resolve_effective_microbatch_batch_size.rs | 15 + .../src/plan_output/main/text.rs | 12 + .../src/plan_output/mod.rs | 5 + .../src/planner_changes/_helpers}/actions.rs | 0 .../src/planner_changes/_helpers}/detect.rs | 30 +- .../src/planner_changes/_helpers}/metadata.rs | 0 .../src/planner_changes/_helpers/mod.rs | 5 + .../src/planner_changes/_helpers}/renames.rs | 0 .../src/planner_changes/_helpers}/results.rs | 0 .../planner_changes/main/detect_changes.rs | 16 + .../main/detect_model_changes.rs | 17 + .../src/planner_changes/main/mod.rs | 14 + .../src/planner_changes/main/model_change.rs | 17 + .../src/planner_changes/main/native_enum.rs | 18 + .../main/plan_reference_renames.rs | 17 + .../src/planner_changes/main/planner_enum.rs | 18 + .../planner_changes/main/register_actions.rs | 12 + .../planner_changes/main/register_detect.rs | 12 + .../planner_changes/main/register_metadata.rs | 12 + .../planner_changes/main/register_renames.rs | 12 + .../main/replaces_incremental_relation.rs | 9 + .../main/resolve_model_actions.rs | 15 + .../main/scope_model_changes.rs | 21 + .../main/without_declared_columns_hash.rs | 9 + .../src/planner_changes/mod.rs | 5 + .../src/planner_changes/models.rs | 14 + .../_helpers/entries}/contexts.rs | 8 +- .../_helpers/entries}/getters.rs | 30 +- .../planner_entries/_helpers/entries/mod.rs | 5 + .../_helpers/entries}/model_entries.rs | 74 ++-- .../_helpers/entries}/relations.rs | 27 +- .../_helpers/entries}/values.rs | 18 +- .../src/planner_entries/_helpers/mod.rs | 2 + .../_helpers/plans}/clone_plans.rs | 34 +- .../_helpers/plans}/command_plans.rs | 29 +- .../_helpers/plans}/display_plan.rs | 15 +- .../_helpers/plans}/execution_plan.rs | 61 +-- .../src/planner_entries/_helpers/plans/mod.rs | 6 + .../_helpers/plans}/plan_output.rs | 29 +- .../_helpers/plans}/test_planning.rs | 4 +- .../src/planner_entries/constants.rs | 26 ++ .../default_microbatch_limit_action.rs | 13 + .../src/planner_entries/main/entries/mod.rs | 6 + .../main/entries/plan_entries.rs | 12 + .../main/entries/plan_model.rs | 17 + .../main/entries/planner_relations_context.rs | 15 + .../main/entries/register_model_entries.rs | 12 + .../main/entries/register_relations.rs | 12 + .../src/planner_entries/main/mod.rs | 2 + .../main/plans/build_audit_command_plan.rs | 17 + .../plans/build_display_only_sqlbuild_plan.rs | 16 + .../main/plans/build_execution_plan.rs | 17 + .../main/plans/build_relation_command_plan.rs | 16 + .../main/plans/build_seed_command_plan.rs | 17 + .../main/plans/build_test_command_plan.rs | 17 + .../src/planner_entries/main/plans/mod.rs | 17 + .../main/plans/plan_output_fields.rs | 15 + .../plans/read_selected_seed_fingerprints.rs | 17 + .../main/plans/register_clone_plans.rs | 12 + .../main/plans/register_command_plans.rs | 12 + .../main/plans/register_display_plan.rs | 12 + .../main/plans/register_execution_plan.rs | 12 + .../main/plans/register_plan_output.rs | 12 + .../main/plans/register_test_planning.rs | 12 + .../plans/resolve_static_command_scope.rs | 16 + .../plans/resolve_static_relation_context.rs | 17 + .../main/plans/timed_clone_planning.rs | 18 + .../src/planner_entries/mod.rs | 7 + .../src/planner_entries/models.rs | 101 +++++ .../src/planner_entries/types.rs | 4 + .../planner_identity/_helpers}/identities.rs | 6 +- .../planner_identity/_helpers}/metadata.rs | 2 +- .../src/planner_identity/_helpers/mod.rs | 3 + .../planner_identity/_helpers}/staleness.rs | 5 +- ...build_compiled_function_fingerprint_sql.rs | 12 + .../main/build_function_local_hashes.rs | 16 + ...ld_model_version_identity_metadata_json.rs | 20 + .../main/build_seed_identity.rs | 12 + .../build_stale_out_of_selection_warnings.rs | 17 + .../main/compiled_function_fingerprint_sql.rs | 12 + .../main/direct_model_version_identities.rs | 22 ++ .../main/function_local_hashes.rs | 17 + .../main/hook_version_hashes.rs | 15 + .../main/merge_recomputed_model_changes.rs | 15 + .../src/planner_identity/main/mod.rs | 18 + .../main/model_declared_columns_hash.rs | 12 + .../model_version_identity_metadata_json.rs | 20 + .../src/planner_identity/main/or_empty.rs | 16 + .../main/register_identities.rs | 12 + .../main/register_metadata.rs | 12 + .../main/register_staleness.rs | 12 + .../planner_identity/main/seed_identity.rs | 12 + .../main/with_honest_model_write_hashes.rs | 17 + .../src/planner_identity/mod.rs | 4 + .../_helpers/column}/column_candidates.rs | 18 +- .../_helpers/column}/column_entries.rs | 34 +- .../_helpers/column}/column_objects.rs | 14 +- .../_helpers/column}/column_overlay.rs | 23 +- .../_helpers/column}/column_planning.rs | 53 ++- .../_helpers/column}/column_warnings.rs | 20 +- .../_helpers/column}/columns.rs | 2 +- .../_helpers/column}/known_input_columns.rs | 0 .../planner_migrations/_helpers/column/mod.rs | 8 + .../_helpers/discovery}/discovery.rs | 56 ++- .../_helpers/discovery}/discovery_events.rs | 20 +- .../_helpers/discovery}/discovery_models.rs | 12 +- .../_helpers/discovery}/discovery_requests.rs | 24 +- .../_helpers/discovery/mod.rs | 5 + .../_helpers/discovery/state_inspection.rs | 111 ++++++ .../src/planner_migrations/_helpers/mod.rs | 6 + .../planner_migrations/_helpers/model/mod.rs | 5 + .../_helpers/model}/model_entries.rs | 28 +- .../_helpers/model}/model_handover.rs | 19 +- .../_helpers/model}/model_overlay.rs | 13 +- .../_helpers/model}/model_planning.rs | 62 +-- .../_helpers/model}/model_warnings.rs | 9 +- .../_helpers/old_name/mod.rs | 5 + .../_helpers/old_name}/old_name_entries.rs | 22 +- .../_helpers/old_name}/old_name_objects.rs | 15 +- .../_helpers/old_name}/old_name_planning.rs | 30 +- .../_helpers/old_name}/old_name_references.rs | 5 +- .../_helpers/old_name}/old_names.rs | 2 +- .../_helpers/planning}/compatibility.rs | 9 +- .../_helpers/planning}/matching.rs | 11 +- .../_helpers/planning/mod.rs | 5 + .../_helpers/planning}/planning_boundary.rs | 16 +- .../_helpers/planning}/planning_execution.rs | 6 +- .../_helpers/planning}/planning_origin.rs | 18 +- .../_helpers/projections}/fingerprint.rs | 0 .../_helpers/projections}/getters.rs | 12 +- .../_helpers/projections/mod.rs | 3 + .../_helpers/projections}/projections.rs | 2 +- .../src/planner_migrations/constants.rs | 38 ++ .../src}/planner_migrations/errors.rs | 2 +- .../planner_migrations/main/fingerprint.rs | 16 + .../main/migration_fingerprint_ref_names.rs | 17 + .../src/planner_migrations/main/mod.rs | 10 + .../main/plan_column_migrations.rs | 16 + .../main/plan_model_migrations.rs | 17 + .../main/plan_old_name_views.rs | 17 + .../main/register_column_planning.rs | 12 + .../main/register_columns.rs | 12 + .../main/register_fingerprint.rs | 12 + .../main/register_projections.rs | 12 + .../main/with_migration_fingerprints.rs | 19 + .../src/planner_migrations/mod.rs | 7 + .../src/planner_migrations/models.rs} | 368 +++++++++++------- .../_helpers/inputs}/contexts.rs | 2 +- .../_helpers/inputs}/getters.rs | 42 +- .../src/planner_phases/_helpers/inputs/mod.rs | 4 + .../_helpers/inputs}/runtime.rs | 5 +- .../_helpers/inputs}/warehouse_state.rs | 28 +- .../src/planner_phases/_helpers/mod.rs | 3 + .../_helpers/outputs}/buildability.rs | 10 +- .../_helpers/outputs}/identities.rs | 24 +- .../planner_phases/_helpers/outputs/mod.rs | 3 + .../_helpers/outputs}/output_assembly.rs | 30 +- .../_helpers/rebuilds}/deferred_origins.rs | 0 .../_helpers/rebuilds}/execution_changes.rs | 27 +- .../_helpers/rebuilds}/full_refresh.rs | 16 +- .../planner_phases/_helpers/rebuilds/mod.rs | 4 + .../_helpers/rebuilds}/retention.rs | 37 +- .../src/planner_phases/constants.rs | 61 +++ .../planner_phases/main/inputs/class_field.rs | 21 + .../inputs/gather_planner_warehouse_state.rs | 17 + .../src/planner_phases/main/inputs/mod.rs | 4 + .../main/inputs/runtime_field.rs | 17 + .../main/inputs/warehouse_field.rs | 17 + .../src/planner_phases/main/mod.rs | 3 + .../main/outputs/add_deferred_origin_notes.rs | 17 + .../main/outputs/add_plan_metadata.rs | 17 + .../main/outputs/add_plan_warnings.rs | 32 ++ .../main/outputs/add_storage_policies.rs | 28 ++ .../main/outputs/base_plan_output.rs | 43 ++ .../outputs/build_planner_identity_context.rs | 20 + .../check_selected_scope_buildability.rs | 17 + .../outputs/detect_stale_warning_changes.rs | 20 + ...etect_stale_warning_changes_of_function.rs | 11 + .../src/planner_phases/main/outputs/mod.rs | 11 + .../planner_identity_work_info_function.rs | 11 + .../outputs/unselected_upstream_models.rs | 12 + .../rebuilds/build_planner_entry_results.rs | 24 ++ .../rebuilds/check_full_rebuild_allowed.rs | 16 + .../check_full_rebuild_allowed_function.rs | 11 + .../rebuilds/defer_to_migration_origins.rs | 21 + .../rebuilds/describe_full_rebuild_cause.rs | 15 + .../describe_full_rebuild_cause_function.rs | 11 + .../main/rebuilds/effective_target_config.rs | 12 + .../effectively_full_refreshed_model_names.rs | 15 + .../src/planner_phases/main/rebuilds/mod.rs | 13 + .../main/rebuilds/plan_retention_function.rs | 11 + .../rebuilds/plan_table_types_function.rs | 11 + .../rebuilds/reconcile_execution_changes.rs | 23 ++ .../rebuilds/resolve_model_full_refresh.rs | 15 + .../rebuilds/table_type_copy_name_function.rs | 11 + .../src/planner_phases/mod.rs | 6 + .../src/planner_phases/models.rs | 108 +++++ .../_helpers/cursor}/cursor_bounds.rs | 14 +- .../_helpers/cursor}/cursor_model.rs | 31 +- .../_helpers/cursor}/cursor_overrides.rs | 5 +- .../_helpers/cursor}/cursor_policies.rs | 0 .../_helpers/cursor}/cursor_ranges.rs | 48 +-- .../_helpers/cursor}/cursor_safety.rs | 14 +- .../_helpers/cursor}/cursor_warnings.rs | 4 +- .../_helpers/cursor}/cursor_windows.rs | 4 +- .../planner_resolution/_helpers/cursor/mod.rs | 8 + .../src/planner_resolution/_helpers/mod.rs | 3 + .../_helpers/references}/locations.rs | 2 +- .../_helpers/references/mod.rs | 5 + .../_helpers/references}/orchestration.rs | 129 +----- .../_helpers/references}/refs.rs | 11 +- .../_helpers/references}/sources.rs | 13 +- .../_helpers/references}/values.rs | 10 +- .../_helpers/safety}/future_safety.rs | 11 +- .../_helpers/safety}/maximum_safety.rs | 21 +- .../planner_resolution/_helpers/safety/mod.rs | 2 + .../src/planner_resolution/constants.rs | 3 + .../main/cursor/config_str.rs | 12 + .../main/cursor/cursor_safety_configs.rs | 15 + .../main/cursor/empty_cursor_inputs.rs | 16 + .../main/cursor/microbatch_range.rs | 16 + .../src/planner_resolution/main/cursor/mod.rs | 16 + .../main/cursor/model_cursor_overrides.rs | 16 + .../cursor/normalize_cursor_snapshot_grain.rs | 16 + .../main/cursor/plan_cursor_bounds.rs | 16 + .../main/cursor/register_cursor_bounds.rs | 12 + .../main/cursor/register_cursor_overrides.rs | 12 + .../main/cursor/register_cursor_ranges.rs | 12 + .../main/cursor/register_cursor_warnings.rs | 12 + .../main/cursor/register_cursor_windows.rs | 12 + .../resolve_effective_timestamp_grain.rs | 22 ++ .../cursor/resolve_start_cursor_config.rs | 15 + .../main/cursor/type_cursor_overrides.rs | 15 + .../cursor/with_cursor_safety_warnings.rs | 16 + .../src/planner_resolution/main/mod.rs | 3 + .../references/apply_deferred_locations.rs | 23 ++ .../references/build_function_locations.rs | 15 + .../main/references/build_model_locations.rs | 13 + .../main/references/build_seed_locations.rs | 13 + .../planner_resolution/main/references/mod.rs | 11 + .../main/references/model_hook_entries.rs | 17 + .../main/references/register_locations.rs | 12 + .../main/references/register_orchestration.rs | 12 + .../main/references/register_refs.rs | 12 + .../main/references/register_sources.rs | 12 + .../main/references/resolve_function_sql.rs | 17 + .../main/references/resolve_model_sql.rs | 18 + .../main/safety/maximum_allowed_start.rs | 16 + .../src/planner_resolution/main/safety/mod.rs | 3 + .../main/safety/register_future_safety.rs | 12 + .../main/safety/register_maximum_safety.rs | 12 + .../src/planner_resolution/mod.rs | 6 + .../src/planner_resolution/models.rs | 168 ++++++++ .../planner_scenario/_helpers}/artifacts.rs | 52 ++- .../src/planner_scenario/_helpers}/cli.rs | 23 +- .../src/planner_scenario/_helpers}/entries.rs | 18 +- .../planner_scenario/_helpers}/execution.rs | 46 ++- .../planner_scenario/_helpers}/fixtures.rs | 28 +- .../src/planner_scenario/_helpers}/graph.rs | 11 +- .../_helpers}/helper_scope.rs | 7 +- .../src/planner_scenario/_helpers/mod.rs | 10 + .../planner_scenario/_helpers}/relations.rs | 19 +- .../src/planner_scenario/_helpers}/sql.rs | 31 +- .../src/planner_scenario/_helpers}/values.rs | 4 +- .../src/planner_scenario/constants.rs | 44 +++ .../main/build_cli_scenario_plan_function.rs | 11 + .../build_scenario_artifact_name_function.rs | 11 + .../build_scenario_execution_plan_function.rs | 11 + .../build_scenario_fixture_plans_function.rs | 11 + .../build_scenario_hash_index_function.rs | 11 + .../build_scenario_relation_map_function.rs | 11 + ...uild_scenario_relation_plan_of_function.rs | 11 + .../compute_scenario_hash_prefix_function.rs | 11 + .../is_scenario_artifact_physical_name.rs | 9 + .../src/planner_scenario/main/mod.rs | 14 + ...cenario_artifact_physical_name_function.rs | 11 + .../main/plan_scenario_graph.rs | 23 ++ .../main/plan_scenario_graph_function.rs | 11 + .../resolve_scenario_check_sql_function.rs | 11 + ...o_artifact_physical_name_check_function.rs | 11 + .../src/planner_scenario/mod.rs | 6 + .../src/planner_scenario/models.rs | 23 ++ .../_helpers}/audit_entries.rs | 13 +- .../src/planner_strategy/_helpers}/audits.rs | 4 +- .../_helpers}/cursor_inputs.rs | 28 +- .../_helpers}/cursor_types.rs | 6 +- .../src/planner_strategy/_helpers}/lineage.rs | 8 +- .../_helpers}/microbatch_limits.rs | 28 +- .../_helpers}/microbatches.rs | 4 +- .../src/planner_strategy/_helpers/mod.rs | 10 + .../_helpers}/model_entries.rs | 30 +- .../planner_strategy/_helpers}/strategy.rs | 17 +- .../src/planner_strategy/_helpers}/values.rs | 2 +- .../main/audit_entries/audit_is_selected.rs | 17 + .../build_selected_audit_entries.rs | 15 + .../main/audit_entries/mod.rs | 3 + .../audit_entries/plan_audit_of_function.rs | 11 + .../src/planner_strategy/main/audits/mod.rs | 1 + .../main/audits/register_audits.rs | 12 + .../build_cursor_input_relations.rs | 18 + .../cursor_input_columns_of_function.rs | 11 + .../cursor_input_relations_of_function.rs | 11 + .../main/cursor_inputs/mod.rs | 6 + .../reject_unexposed_cursor_input_columns.rs | 17 + .../runtime_cursor_producer_names.rs | 16 + .../runtime_cursor_producer_names_function.rs | 11 + .../cursor_types/classify_cursor_sql_type.rs | 16 + .../planner_strategy/main/cursor_types/mod.rs | 2 + .../cursor_types/register_cursor_types.rs | 12 + .../src/planner_strategy/main/lineage/mod.rs | 2 + .../lineage/reject_capped_watermark_inputs.rs | 17 + .../main/lineage/resolve_lineage_reference.rs | 18 + .../apply_configured_microbatch_limits.rs | 16 + ...y_configured_microbatch_limits_function.rs | 11 + .../apply_microbatch_limit_function.rs | 11 + .../main/microbatch_limits/mod.rs | 3 + .../planner_strategy/main/microbatches/mod.rs | 1 + .../microbatches/register_microbatches.rs | 12 + .../src/planner_strategy/main/mod.rs | 10 + .../assemble_model_plan_entry.rs | 16 + .../model_entries/extract_seed_columns.rs | 13 + .../main/model_entries/mod.rs | 4 + .../resolve_microbatch_lookback.rs | 15 + .../resolve_microbatch_lookback_function.rs | 11 + .../main/strategy/materialization_of.rs | 12 + .../src/planner_strategy/main/strategy/mod.rs | 3 + .../main/strategy/register_strategy.rs | 12 + .../strategy/resolve_model_plan_action.rs | 17 + .../main/values/input_error.rs | 12 + .../src/planner_strategy/main/values/mod.rs | 2 + .../src/planner_strategy/main/values/text.rs | 12 + .../src/planner_strategy/mod.rs | 4 + .../_helpers/inspection}/cursors.rs | 31 +- .../_helpers/inspection/fingerprint_cache.rs | 112 ++++++ .../_helpers/inspection/mod.rs | 4 + .../_helpers/inspection}/snapshot.rs | 44 ++- .../_helpers/inspection}/watermarks.rs | 15 +- .../src/planner_warehouse/_helpers/mod.rs | 2 + .../planner_warehouse/_helpers/sources/mod.rs | 5 + .../_helpers/sources}/semantic_sources.rs | 19 +- .../_helpers/sources}/source_columns.rs | 18 +- .../_helpers/sources}/source_deferral.rs | 5 +- .../_helpers/sources}/source_freshness.rs | 25 +- .../_helpers/sources}/source_tables.rs | 13 +- .../src}/planner_warehouse/constants.rs | 31 +- .../main/inspection/adapter_cursor_sql.rs | 16 + .../main/inspection/bound_task.rs | 17 + .../gather_redirected_cursor_snapshots.rs | 16 + .../gather_warehouse_snapshot_function.rs | 11 + ...igration_fingerprint_work_info_function.rs | 11 + .../planner_warehouse/main/inspection/mod.rs | 8 + .../planner_metadata_name_filter_function.rs | 11 + .../main/inspection/warehouse_snapshot.rs | 13 + .../watermark_type_is_compatible_function.rs | 11 + .../src/planner_warehouse/main/mod.rs | 2 + .../build_planner_source_freshness_result.rs | 22 ++ ...ner_source_freshness_result_of_function.rs | 11 + .../main/sources/build_source_read_map.rs | 16 + .../check_selected_source_tables_exist.rs | 22 ++ ...ed_source_tables_exist_binding_function.rs | 11 + .../main/sources/gather_source_columns.rs | 18 + .../sources/gather_source_columns_function.rs | 11 + .../main/sources/gather_source_inspection.rs | 17 + .../list_source_candidates_function.rs | 11 + .../src/planner_warehouse/main/sources/mod.rs | 13 + .../main/sources/semantic_source_columns.rs | 16 + .../semantic_source_columns_function.rs | 11 + .../source_freshness_unknown_warnings.rs | 14 + .../sources/with_declared_source_reads.rs | 17 + .../src/planner_warehouse/mod.rs | 6 + .../src/planner_warehouse/models.rs} | 203 ++++------ crates/sqlbuild-python/Cargo.toml | 3 +- .../_helpers/audit_command/command.rs | 2 +- .../_helpers/build_command/command.rs | 5 +- .../_helpers/build_command/execution.rs | 17 +- .../_helpers/build_command/planning.rs | 2 +- .../_helpers/build_progress/plan_facts.rs | 2 +- .../_helpers/build_progress/result_text.rs | 3 +- .../change_selection/definition_snapshot.rs | 3 +- .../_helpers/change_selection/selection.rs | 2 +- .../bindings/_helpers/check/check_command.rs | 2 +- .../bindings/_helpers/clone/clone_command.rs | 2 +- .../bindings/_helpers/clone/clone_pipeline.rs | 3 +- .../_helpers/compile_driver/plan_result.rs | 2 +- .../_helpers/compile_driver/static_context.rs | 5 +- .../src/bindings/_helpers/dbt_interop/plan.rs | 2 +- .../_helpers/dbt_interop/plan_output.rs | 4 +- .../_helpers/freshness/freshness_command.rs | 8 +- .../src/bindings/_helpers/freshness/mod.rs | 7 - .../_helpers/freshness/registration.rs | 21 +- .../janitor/janitor_classification.rs | 2 +- .../bindings/_helpers/lifecycle_events/mod.rs | 4 - .../_helpers/lifecycle_events/registration.rs | 10 +- .../bindings/_helpers/load/load_command.rs | 2 +- .../_helpers/location_command/snapshot.rs | 2 +- .../src/bindings/_helpers/microbatches/mod.rs | 3 - .../_helpers/microbatches/registration.rs | 7 +- .../src/bindings/_helpers/mod.rs | 2 - .../native_host/warehouse_metadata.rs | 4 +- .../bindings/_helpers/plan_command/changed.rs | 6 +- .../bindings/_helpers/plan_command/plan.rs | 11 +- .../src/bindings/_helpers/plan_output/mod.rs | 8 - .../_helpers/plan_output/registration.rs | 16 +- .../bindings/_helpers/planner_changes/mod.rs | 5 - .../_helpers/planner_changes/registration.rs | 13 +- .../bindings/_helpers/planner_entries/mod.rs | 12 - .../_helpers/planner_entries/models.rs | 99 ----- .../_helpers/planner_entries/registration.rs | 31 +- .../bindings/_helpers/planner_identity/mod.rs | 3 - .../_helpers/planner_identity/registration.rs | 10 +- .../_helpers/planner_migrations/constants.rs | 14 - .../_helpers/planner_migrations/mod.rs | 34 -- .../_helpers/planner_migrations/models.rs | 150 ------- .../planner_migrations/registration.rs | 16 +- .../_helpers/planner_phases/constants.rs | 14 - .../bindings/_helpers/planner_phases/mod.rs | 13 - .../_helpers/planner_phases/models.rs | 107 ----- .../_helpers/planner_phases/registration.rs | 50 +-- .../_helpers/planner_resolution/constants.rs | 3 - .../_helpers/planner_resolution/mod.rs | 17 - .../_helpers/planner_resolution/models.rs | 24 -- .../planner_resolution/registration.rs | 33 +- .../_helpers/planner_scenario/constants.rs | 24 -- .../bindings/_helpers/planner_scenario/mod.rs | 12 - .../_helpers/planner_scenario/models.rs | 22 -- .../_helpers/planner_scenario/registration.rs | 67 ++-- .../bindings/_helpers/planner_strategy/mod.rs | 10 - .../_helpers/planner_strategy/registration.rs | 57 ++- .../_helpers/planner_warehouse/mod.rs | 11 - .../_helpers/planner_warehouse/models.rs | 13 - .../planner_warehouse/registration.rs | 59 +-- .../_helpers/record_behaviours/constants.rs | 11 +- .../_helpers/record_behaviours/freshness.rs | 2 +- .../bindings/_helpers/result_output/assets.rs | 7 +- .../bindings/_helpers/result_output/checks.rs | 5 +- .../_helpers/result_output/documents.rs | 5 +- .../result_output/integration_records.rs | 5 +- .../_helpers/result_output/projection.rs | 2 +- .../bindings/_helpers/seed_command/command.rs | 4 +- .../_helpers/sql_test_command/test_command.rs | 2 +- fensu.toml | 338 ++++++++-------- scripts/fensu_policy/constants.py | 1 + scripts/native_module_paths/constants.py | 1 + 547 files changed, 7073 insertions(+), 2543 deletions(-) create mode 100644 crates/sqlbuild-python-planner/Cargo.toml rename crates/{sqlbuild-python/src/bindings/_helpers/consumption => sqlbuild-python-planner/src/consumption/_helpers}/facts.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/consumption => sqlbuild-python-planner/src/consumption/_helpers}/mod.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/consumption => sqlbuild-python-planner/src/consumption/_helpers}/policy.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/consumption => sqlbuild-python-planner/src/consumption/_helpers}/recorder.rs (60%) rename crates/{sqlbuild-python/src/bindings/_helpers/consumption => sqlbuild-python-planner/src/consumption/_helpers}/relation_reads.rs (100%) create mode 100644 crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs create mode 100644 crates/sqlbuild-python-planner/src/consumption/main/held_back_text.rs create mode 100644 crates/sqlbuild-python-planner/src/consumption/main/kind_value.rs create mode 100644 crates/sqlbuild-python-planner/src/consumption/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/consumption/main/policy_line.rs create mode 100644 crates/sqlbuild-python-planner/src/consumption/main/prepare.rs create mode 100644 crates/sqlbuild-python-planner/src/consumption/main/preview_changed.rs create mode 100644 crates/sqlbuild-python-planner/src/consumption/main/reach_value.rs create mode 100644 crates/sqlbuild-python-planner/src/consumption/main/relation.rs create mode 100644 crates/sqlbuild-python-planner/src/consumption/main/run_reason_text.rs create mode 100644 crates/sqlbuild-python-planner/src/consumption/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/consumption/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/freshness => sqlbuild-python-planner/src/freshness/_helpers}/exception_types.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/freshness => sqlbuild-python-planner/src/freshness/_helpers}/freshness_observe.rs (97%) create mode 100644 crates/sqlbuild-python-planner/src/freshness/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/freshness => sqlbuild-python-planner/src/freshness/_helpers}/observation.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/freshness => sqlbuild-python-planner/src/freshness/_helpers}/planning.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/freshness => sqlbuild-python-planner/src/freshness/_helpers}/publication.rs (99%) rename crates/{sqlbuild-python/src/bindings/_helpers/freshness => sqlbuild-python-planner/src/freshness/_helpers}/state_io.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/freshness => sqlbuild-python-planner/src/freshness/_helpers}/support.rs (94%) create mode 100644 crates/sqlbuild-python-planner/src/freshness/constants.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/append_freshness_records.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/build_direct_source_freshness_planning_result_function.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/build_direct_source_freshness_propagation_result_function.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/normalize_observation.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/observe_freshness.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/presumed_utc_datetime.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/read_latest.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/register_exception_types.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/register_freshness_observe.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/register_observation.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/register_planning.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/register_publication.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/require_aware_utc_datetime.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/main/row_record.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/freshness/models.rs create mode 100644 crates/sqlbuild-python-planner/src/lib.rs rename crates/{sqlbuild-python/src/bindings/_helpers/lifecycle_events => sqlbuild-python-planner/src/lifecycle_events/_helpers}/catalog.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/lifecycle_events => sqlbuild-python-planner/src/lifecycle_events/_helpers}/envelopes.rs (95%) create mode 100644 crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/lifecycle_events => sqlbuild-python-planner/src/lifecycle_events/_helpers}/payloads.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/lifecycle_events => sqlbuild-python-planner/src/lifecycle_events/_helpers}/validation.rs (94%) create mode 100644 crates/sqlbuild-python-planner/src/lifecycle_events/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/lifecycle_events/main/register_catalog.rs create mode 100644 crates/sqlbuild-python-planner/src/lifecycle_events/main/register_envelopes.rs create mode 100644 crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs create mode 100644 crates/sqlbuild-python-planner/src/lifecycle_events/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/microbatches => sqlbuild-python-planner/src/microbatches/_helpers}/conversion.rs (100%) create mode 100644 crates/sqlbuild-python-planner/src/microbatches/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/microbatches => sqlbuild-python-planner/src/microbatches/_helpers}/projection.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/microbatches => sqlbuild-python-planner/src/microbatches/_helpers}/statements.rs (99%) create mode 100644 crates/sqlbuild-python-planner/src/microbatches/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs create mode 100644 crates/sqlbuild-python-planner/src/microbatches/main/register_statements.rs create mode 100644 crates/sqlbuild-python-planner/src/microbatches/main/timestamp.rs create mode 100644 crates/sqlbuild-python-planner/src/microbatches/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/observations => sqlbuild-python-planner/src/observations/_helpers}/mod.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/observations => sqlbuild-python-planner/src/observations/_helpers}/publication.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/observations => sqlbuild-python-planner/src/observations/_helpers}/source_capture.rs (100%) create mode 100644 crates/sqlbuild-python-planner/src/observations/main/append_build_observations.rs create mode 100644 crates/sqlbuild-python-planner/src/observations/main/arm_source_capture.rs create mode 100644 crates/sqlbuild-python-planner/src/observations/main/capture_source_columns.rs create mode 100644 crates/sqlbuild-python-planner/src/observations/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/observations/main/take_source_capture.rs create mode 100644 crates/sqlbuild-python-planner/src/observations/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/plan_output => sqlbuild-python-planner/src/plan_output/_helpers}/cursor_details.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/plan_output => sqlbuild-python-planner/src/plan_output/_helpers}/display.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/plan_output => sqlbuild-python-planner/src/plan_output/_helpers}/migrations.rs (96%) create mode 100644 crates/sqlbuild-python-planner/src/plan_output/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/plan_output => sqlbuild-python-planner/src/plan_output/_helpers}/plan_json.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/plan_output => sqlbuild-python-planner/src/plan_output/_helpers}/plan_text.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/plan_output => sqlbuild-python-planner/src/plan_output/_helpers}/python_identity.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/plan_output => sqlbuild-python-planner/src/plan_output/_helpers}/safety.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/plan_output => sqlbuild-python-planner/src/plan_output/_helpers}/values.rs (100%) create mode 100644 crates/sqlbuild-python-planner/src/plan_output/constants.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/aligned_timestamp_bounds.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/any.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/cursor_bound_display.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/dict.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/format_plan.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/format_plan_json.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/future_cursor_safety.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/inclusive_cursor_end.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/items.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/maximum_start_safety.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/optional_facts.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/register_migrations.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/register_plan_json.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/register_plan_text.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/register_safety.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/resolve_bounded_cursor_override.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/resolve_effective_microbatch_batch_size.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/text.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_changes => sqlbuild-python-planner/src/planner_changes/_helpers}/actions.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_changes => sqlbuild-python-planner/src/planner_changes/_helpers}/detect.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_changes => sqlbuild-python-planner/src/planner_changes/_helpers}/metadata.rs (100%) create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_changes => sqlbuild-python-planner/src/planner_changes/_helpers}/renames.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_changes => sqlbuild-python-planner/src/planner_changes/_helpers}/results.rs (100%) create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/main/detect_model_changes.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/main/model_change.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/main/native_enum.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/main/plan_reference_renames.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/main/planner_enum.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/main/register_actions.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/main/register_renames.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/main/replaces_incremental_relation.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/main/resolve_model_actions.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/main/scope_model_changes.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/main/without_declared_columns_hash.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_changes/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_entries => sqlbuild-python-planner/src/planner_entries/_helpers/entries}/contexts.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_entries => sqlbuild-python-planner/src/planner_entries/_helpers/entries}/getters.rs (82%) create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_entries => sqlbuild-python-planner/src/planner_entries/_helpers/entries}/model_entries.rs (87%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_entries => sqlbuild-python-planner/src/planner_entries/_helpers/entries}/relations.rs (89%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_entries => sqlbuild-python-planner/src/planner_entries/_helpers/entries}/values.rs (84%) create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_entries => sqlbuild-python-planner/src/planner_entries/_helpers/plans}/clone_plans.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_entries => sqlbuild-python-planner/src/planner_entries/_helpers/plans}/command_plans.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_entries => sqlbuild-python-planner/src/planner_entries/_helpers/plans}/display_plan.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_entries => sqlbuild-python-planner/src/planner_entries/_helpers/plans}/execution_plan.rs (80%) create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_entries => sqlbuild-python-planner/src/planner_entries/_helpers/plans}/plan_output.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_entries => sqlbuild-python-planner/src/planner_entries/_helpers/plans}/test_planning.rs (98%) create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/constants.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/entries/default_microbatch_limit_action.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/entries/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_entries.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_model.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/entries/planner_relations_context.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_model_entries.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_relations.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_display_only_sqlbuild_plan.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_relation_command_plan.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_seed_command_plan.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_test_command_plan.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/plan_output_fields.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/read_selected_seed_fingerprints.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_clone_plans.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_command_plans.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_display_plan.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_execution_plan.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_plan_output.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_test_planning.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_command_scope.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/main/plans/timed_clone_planning.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/models.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_entries/types.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_identity => sqlbuild-python-planner/src/planner_identity/_helpers}/identities.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_identity => sqlbuild-python-planner/src/planner_identity/_helpers}/metadata.rs (99%) create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_identity => sqlbuild-python-planner/src/planner_identity/_helpers}/staleness.rs (98%) create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/build_compiled_function_fingerprint_sql.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/build_function_local_hashes.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/build_model_version_identity_metadata_json.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/build_seed_identity.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/build_stale_out_of_selection_warnings.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/compiled_function_fingerprint_sql.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/direct_model_version_identities.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/function_local_hashes.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/hook_version_hashes.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/merge_recomputed_model_changes.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/model_declared_columns_hash.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/model_version_identity_metadata_json.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/or_empty.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/register_identities.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/register_metadata.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/register_staleness.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/seed_identity.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/main/with_honest_model_write_hashes.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_identity/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/column}/column_candidates.rs (88%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/column}/column_entries.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/column}/column_objects.rs (88%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/column}/column_overlay.rs (89%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/column}/column_planning.rs (71%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/column}/column_warnings.rs (87%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/column}/columns.rs (98%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/column}/known_input_columns.rs (100%) create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/discovery}/discovery.rs (66%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/discovery}/discovery_events.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/discovery}/discovery_models.rs (91%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/discovery}/discovery_requests.rs (89%) create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/model}/model_entries.rs (89%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/model}/model_handover.rs (88%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/model}/model_overlay.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/model}/model_planning.rs (81%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/model}/model_warnings.rs (88%) create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/old_name}/old_name_entries.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/old_name}/old_name_objects.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/old_name}/old_name_planning.rs (87%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/old_name}/old_name_references.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/old_name}/old_names.rs (99%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/planning}/compatibility.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/planning}/matching.rs (94%) create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/planning}/planning_boundary.rs (83%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/planning}/planning_execution.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/planning}/planning_origin.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/projections}/fingerprint.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/projections}/getters.rs (66%) create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations => sqlbuild-python-planner/src/planner_migrations/_helpers/projections}/projections.rs (98%) create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/constants.rs rename crates/{sqlbuild-python/src/bindings/_helpers => sqlbuild-python-planner/src}/planner_migrations/errors.rs (69%) create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/main/fingerprint.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/main/migration_fingerprint_ref_names.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/main/plan_column_migrations.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/main/plan_model_migrations.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/main/plan_old_name_views.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/main/register_column_planning.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/main/register_columns.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/main/register_fingerprint.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/main/register_projections.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs => sqlbuild-python-planner/src/planner_migrations/models.rs} (67%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_phases => sqlbuild-python-planner/src/planner_phases/_helpers/inputs}/contexts.rs (91%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_phases => sqlbuild-python-planner/src/planner_phases/_helpers/inputs}/getters.rs (76%) create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_phases => sqlbuild-python-planner/src/planner_phases/_helpers/inputs}/runtime.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_phases => sqlbuild-python-planner/src/planner_phases/_helpers/inputs}/warehouse_state.rs (87%) create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_phases => sqlbuild-python-planner/src/planner_phases/_helpers/outputs}/buildability.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_phases => sqlbuild-python-planner/src/planner_phases/_helpers/outputs}/identities.rs (88%) create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_phases => sqlbuild-python-planner/src/planner_phases/_helpers/outputs}/output_assembly.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_phases => sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds}/deferred_origins.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_phases => sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds}/execution_changes.rs (86%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_phases => sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds}/full_refresh.rs (84%) create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_phases => sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds}/retention.rs (94%) create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/constants.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/inputs/class_field.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/inputs/gather_planner_warehouse_state.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/inputs/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_field.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/inputs/warehouse_field.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_deferred_origin_notes.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_metadata.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_warnings.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_storage_policies.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/outputs/base_plan_output.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/outputs/build_planner_identity_context.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/outputs/check_selected_scope_buildability.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes_of_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/outputs/planner_identity_work_info_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/outputs/unselected_upstream_models.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/build_planner_entry_results.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/defer_to_migration_origins.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effective_target_config.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_retention_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_table_types_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/reconcile_execution_changes.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/resolve_model_full_refresh.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/table_type_copy_name_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_resolution => sqlbuild-python-planner/src/planner_resolution/_helpers/cursor}/cursor_bounds.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_resolution => sqlbuild-python-planner/src/planner_resolution/_helpers/cursor}/cursor_model.rs (89%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_resolution => sqlbuild-python-planner/src/planner_resolution/_helpers/cursor}/cursor_overrides.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_resolution => sqlbuild-python-planner/src/planner_resolution/_helpers/cursor}/cursor_policies.rs (100%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_resolution => sqlbuild-python-planner/src/planner_resolution/_helpers/cursor}/cursor_ranges.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_resolution => sqlbuild-python-planner/src/planner_resolution/_helpers/cursor}/cursor_safety.rs (89%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_resolution => sqlbuild-python-planner/src/planner_resolution/_helpers/cursor}/cursor_warnings.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_resolution => sqlbuild-python-planner/src/planner_resolution/_helpers/cursor}/cursor_windows.rs (98%) create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_resolution => sqlbuild-python-planner/src/planner_resolution/_helpers/references}/locations.rs (95%) create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_resolution => sqlbuild-python-planner/src/planner_resolution/_helpers/references}/orchestration.rs (66%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_resolution => sqlbuild-python-planner/src/planner_resolution/_helpers/references}/refs.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_resolution => sqlbuild-python-planner/src/planner_resolution/_helpers/references}/sources.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_resolution => sqlbuild-python-planner/src/planner_resolution/_helpers/references}/values.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_resolution => sqlbuild-python-planner/src/planner_resolution/_helpers/safety}/future_safety.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_resolution => sqlbuild-python-planner/src/planner_resolution/_helpers/safety}/maximum_safety.rs (91%) create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/constants.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/config_str.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/cursor_safety_configs.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/empty_cursor_inputs.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/microbatch_range.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/model_cursor_overrides.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/normalize_cursor_snapshot_grain.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/plan_cursor_bounds.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_bounds.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_overrides.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_warnings.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_windows.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_effective_timestamp_grain.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_start_cursor_config.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/type_cursor_overrides.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/with_cursor_safety_warnings.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/references/apply_deferred_locations.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_function_locations.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_model_locations.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_seed_locations.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/references/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/references/model_hook_entries.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_sources.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_function_sql.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_model_sql.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/safety/maximum_allowed_start.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/safety/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_future_safety.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_maximum_safety.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_resolution/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_scenario => sqlbuild-python-planner/src/planner_scenario/_helpers}/artifacts.rs (80%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_scenario => sqlbuild-python-planner/src/planner_scenario/_helpers}/cli.rs (88%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_scenario => sqlbuild-python-planner/src/planner_scenario/_helpers}/entries.rs (88%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_scenario => sqlbuild-python-planner/src/planner_scenario/_helpers}/execution.rs (90%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_scenario => sqlbuild-python-planner/src/planner_scenario/_helpers}/fixtures.rs (87%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_scenario => sqlbuild-python-planner/src/planner_scenario/_helpers}/graph.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_scenario => sqlbuild-python-planner/src/planner_scenario/_helpers}/helper_scope.rs (92%) create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_scenario => sqlbuild-python-planner/src/planner_scenario/_helpers}/relations.rs (91%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_scenario => sqlbuild-python-planner/src/planner_scenario/_helpers}/sql.rs (84%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_scenario => sqlbuild-python-planner/src/planner_scenario/_helpers}/values.rs (90%) create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/constants.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/main/build_cli_scenario_plan_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_artifact_name_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_execution_plan_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_fixture_plans_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_hash_index_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_map_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_plan_of_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/main/compute_scenario_hash_prefix_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/main/is_scenario_artifact_physical_name.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/main/parse_scenario_artifact_physical_name_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/main/resolve_scenario_check_sql_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/main/scenario_artifact_physical_name_check_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_scenario/models.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-planner/src/planner_strategy/_helpers}/audit_entries.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-planner/src/planner_strategy/_helpers}/audits.rs (97%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-planner/src/planner_strategy/_helpers}/cursor_inputs.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-planner/src/planner_strategy/_helpers}/cursor_types.rs (95%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-planner/src/planner_strategy/_helpers}/lineage.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-planner/src/planner_strategy/_helpers}/microbatch_limits.rs (89%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-planner/src/planner_strategy/_helpers}/microbatches.rs (98%) create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-planner/src/planner_strategy/_helpers}/model_entries.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-planner/src/planner_strategy/_helpers}/strategy.rs (92%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_strategy => sqlbuild-python-planner/src/planner_strategy/_helpers}/values.rs (98%) create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/plan_audit_of_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/audits/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/build_cursor_input_relations.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_columns_of_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_relations_of_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/reject_unexposed_cursor_input_columns.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/classify_cursor_sql_type.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/register_cursor_types.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/reject_capped_watermark_inputs.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/resolve_lineage_reference.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_microbatch_limit_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/register_microbatches.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/assemble_model_plan_entry.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/extract_seed_columns.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/materialization_of.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/resolve_model_plan_action.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/values/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/main/values/text.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_strategy/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_warehouse => sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection}/cursors.rs (97%) create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_warehouse => sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection}/snapshot.rs (96%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_warehouse => sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection}/watermarks.rs (91%) create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_warehouse => sqlbuild-python-planner/src/planner_warehouse/_helpers/sources}/semantic_sources.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_warehouse => sqlbuild-python-planner/src/planner_warehouse/_helpers/sources}/source_columns.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_warehouse => sqlbuild-python-planner/src/planner_warehouse/_helpers/sources}/source_deferral.rs (99%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_warehouse => sqlbuild-python-planner/src/planner_warehouse/_helpers/sources}/source_freshness.rs (88%) rename crates/{sqlbuild-python/src/bindings/_helpers/planner_warehouse => sqlbuild-python-planner/src/planner_warehouse/_helpers/sources}/source_tables.rs (94%) rename crates/{sqlbuild-python/src/bindings/_helpers => sqlbuild-python-planner/src}/planner_warehouse/constants.rs (88%) create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/adapter_cursor_sql.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/bound_task.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_redirected_cursor_snapshots.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_warehouse_snapshot_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/migration_fingerprint_work_info_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/planner_metadata_name_filter_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/warehouse_snapshot.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/watermark_type_is_compatible_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result_of_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_source_read_map.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist_binding_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_inspection.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/list_source_candidates_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/mod.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns_function.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/source_freshness_unknown_warnings.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/with_declared_source_reads.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_warehouse/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs => sqlbuild-python-planner/src/planner_warehouse/models.rs} (67%) delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/constants.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/models.rs diff --git a/Cargo.lock b/Cargo.lock index 012df2cbaf..df82352b73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4450,7 +4450,6 @@ dependencies = [ "sqlbuild-cli", "sqlbuild-config", "sqlbuild-core", - "sqlbuild-cursor", "sqlbuild-discovery", "sqlbuild-host", "sqlbuild-model-config", @@ -4459,6 +4458,7 @@ dependencies = [ "sqlbuild-pysource", "sqlbuild-python-base", "sqlbuild-python-front", + "sqlbuild-python-planner", "sqlbuild-python-project", "sqlbuild-recall", "sqlbuild-refactor", @@ -4468,7 +4468,6 @@ dependencies = [ "sqlbuild-sqltext", "sqlbuild-state", "sqlclean", - "uuid", ] [[package]] @@ -4525,6 +4524,29 @@ dependencies = [ "uuid", ] +[[package]] +name = "sqlbuild-python-planner" +version = "0.0.0" +dependencies = [ + "pyo3", + "serde_json", + "sqlbuild-analysis", + "sqlbuild-cache", + "sqlbuild-core", + "sqlbuild-cursor", + "sqlbuild-discovery", + "sqlbuild-host", + "sqlbuild-model-config", + "sqlbuild-project", + "sqlbuild-python-base", + "sqlbuild-python-front", + "sqlbuild-python-project", + "sqlbuild-rules", + "sqlbuild-sqltext", + "sqlbuild-state", + "uuid", +] + [[package]] name = "sqlbuild-python-project" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 23c523a3cb..51d7ce344b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ members = [ "crates/sqlbuild-python-base", "crates/sqlbuild-python-front", "crates/sqlbuild-python-project", + "crates/sqlbuild-python-planner", "crates/sqlbuild-python", ] resolver = "2" @@ -61,9 +62,10 @@ order = [ "sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", + "sqlbuild-python-planner", "sqlbuild-python", ] -python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python"] +python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python-planner", "sqlbuild-python"] polyglot-floor = "sqlbuild-analysis" [workspace.package] @@ -186,6 +188,7 @@ sqlbuild-project = { path = "crates/sqlbuild-project" } sqlbuild-python-base = { path = "crates/sqlbuild-python-base" } sqlbuild-python-front = { path = "crates/sqlbuild-python-front" } sqlbuild-python-project = { path = "crates/sqlbuild-python-project" } +sqlbuild-python-planner = { path = "crates/sqlbuild-python-planner" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } sqlbuild-recall = { path = "crates/sqlbuild-recall" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } diff --git a/crates/sqlbuild-python-planner/Cargo.toml b/crates/sqlbuild-python-planner/Cargo.toml new file mode 100644 index 0000000000..2495999310 --- /dev/null +++ b/crates/sqlbuild-python-planner/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "sqlbuild-python-planner" +description = "The plan layer of the sqlbuild._native bindings: execution plans and their entries, planner phases, changes, migrations, resolution, scenarios, strategy and warehouse state, freshness, consumption, microbatches, observations, lifecycle events and plan output." +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +pyo3.workspace = true +serde_json.workspace = true +sqlbuild-analysis.workspace = true +sqlbuild-cache.workspace = true +sqlbuild-core.workspace = true +sqlbuild-cursor.workspace = true +sqlbuild-discovery.workspace = true +sqlbuild-host.workspace = true +sqlbuild-model-config.workspace = true +sqlbuild-project.workspace = true +sqlbuild-python-base.workspace = true +sqlbuild-python-front.workspace = true +sqlbuild-python-project.workspace = true +sqlbuild-rules.workspace = true +sqlbuild-sqltext.workspace = true +sqlbuild-state.workspace = true +uuid.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/facts.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs rename to crates/sqlbuild-python-planner/src/consumption/_helpers/facts.rs index ebd65842a7..c04e35d4a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs +++ b/crates/sqlbuild-python-planner/src/consumption/_helpers/facts.rs @@ -7,13 +7,13 @@ use pyo3::types::PyDict; use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; use sqlbuild_state::consumption::models::{ForcedRun, InputFact, NodeFacts, NodeKey, RelationKey}; -use crate::bindings::_helpers::consumption::relation_reads::{ - Locations, relation, sql_reads_undeclared, -}; -use crate::bindings::_helpers::planner_identity::identities::hook_version_hashes; -use crate::bindings::_helpers::planner_identity::metadata::{ - function_local_hashes, model_version_identity_metadata_json, seed_identity, -}; +use crate::consumption::_helpers::relation_reads::Locations; +use crate::consumption::_helpers::relation_reads::relation; +use crate::consumption::_helpers::relation_reads::sql_reads_undeclared; +use crate::planner_identity::main::function_local_hashes::function_local_hashes; +use crate::planner_identity::main::hook_version_hashes::hook_version_hashes; +use crate::planner_identity::main::model_version_identity_metadata_json::model_version_identity_metadata_json; +use crate::planner_identity::main::seed_identity::seed_identity; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_project::compiled_views::main::compiled_models::dep_rows::dep_rows; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/mod.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs rename to crates/sqlbuild-python-planner/src/consumption/_helpers/mod.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/policy.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs rename to crates/sqlbuild-python-planner/src/consumption/_helpers/policy.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/recorder.rs similarity index 60% rename from crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs rename to crates/sqlbuild-python-planner/src/consumption/_helpers/recorder.rs index ecf1b8c59c..255f24fb13 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs +++ b/crates/sqlbuild-python-planner/src/consumption/_helpers/recorder.rs @@ -1,50 +1,50 @@ //! Record what each model and seed execution consumed, and decide `--changed` skips, around one build. +use crate::consumption::models::ChangedDecision; +use crate::consumption::models::ConsumptionRecorder; +use crate::consumption::models::Prepared; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Mutex; use std::time::{Duration, Instant}; -use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::{pyclass, pymethods}; use sqlbuild_state::consumption::constants::RECENT_EXECUTIONS; use sqlbuild_state::consumption::main::changed_run_set::changed_run_set; -use sqlbuild_state::consumption::main::consumed_inputs::consumed_inputs; use sqlbuild_state::consumption::main::consumption_history::consumption_history; use sqlbuild_state::consumption::main::read_back_history::read_back_history; use sqlbuild_state::consumption::main::started_append::started_append; -use sqlbuild_state::consumption::main::succeeded_appends::succeeded_appends; use sqlbuild_state::consumption::models::{ ChangedOutcome, ChangedPolicy, ConsumptionAppend, ConsumptionHistory, ConsumptionRow, ForcedRun, HeldBack, InputFact, NodeFacts, NodeKey, Reach, RelationKey, RunReason, }; use uuid::Uuid; -use crate::bindings::_helpers::consumption::facts::{ - BuildFacts, MODEL, SEED, build_facts, enum_text, -}; -use sqlbuild_python_base::records::main::fields::field::field as native_field; +use crate::consumption::_helpers::facts::BuildFacts; +use crate::consumption::_helpers::facts::MODEL; +use crate::consumption::_helpers::facts::SEED; +use crate::consumption::_helpers::facts::build_facts; use sqlbuild_python_front::native_host::models::{ ConsumptionConnection, Location, NodeCompleteHook, }; /// Succeeded rows are flushed once this many executions are pending. -const FLUSH_EXECUTIONS: usize = 100; +pub(crate) const FLUSH_EXECUTIONS: usize = 100; /// Or once this long has passed since the last flush. -const FLUSH_INTERVAL: Duration = Duration::from_secs(30); +pub(crate) const FLUSH_INTERVAL: Duration = Duration::from_secs(30); const MODEL_RESULT: &str = "model_name"; const SEED_RESULT: &str = "seed_name"; -const SUCCESS: &str = "success"; -const ERROR_OUTCOME: &str = "error"; +pub(crate) const SUCCESS: &str = "success"; +pub(crate) const ERROR_OUTCOME: &str = "error"; /// The mutable side of one build's recording. -struct Recording { - pending: BTreeMap>, - pending_executions: usize, - pending_seeds: Vec, - tables: BTreeSet, - last_flush: Instant, - warnings: Vec, +#[derive(Debug)] +pub(crate) struct Recording { + pub(crate) pending: BTreeMap>, + pub(crate) pending_executions: usize, + pub(crate) pending_seeds: Vec, + pub(crate) tables: BTreeSet, + pub(crate) last_flush: Instant, + pub(crate) warnings: Vec, } impl Default for Recording { @@ -60,40 +60,7 @@ impl Default for Recording { } } -/// Appends `started` rows before a build and `succeeded` rows as its models and seeds complete. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct ConsumptionRecorder { - connection: ConsumptionConnection, - run_id: String, - nodes: HashMap, - history: ConsumptionHistory, - executions: HashMap, - failed_start: HashMap, - base_on_node_complete: NodeCompleteHook, - recording: Mutex, -} - -/// What `--changed` decided, for the build's report. -pub(crate) struct ChangedDecision { - pub(crate) policy: ChangedPolicy, - pub(crate) run: usize, - /// Unchanged nodes, which are up to date. - pub(crate) skipped: Vec, - /// Nodes downstream of a change that wait for a later run. - pub(crate) held_back: Vec<(NodeKey, HeldBack)>, - pub(crate) reasons: Vec<(NodeKey, RunReason)>, -} - -impl ChangedDecision { - /// Every selected node `--changed` leaves unbuilt: unchanged and held back. - pub(crate) fn not_run(&self) -> impl Iterator { - self.skipped - .iter() - .chain(self.held_back.iter().map(|(key, _)| key)) - } -} - -fn kind_of(result: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn kind_of(result: &Bound<'_, PyAny>) -> PyResult> { for (field, kind) in [(MODEL_RESULT, MODEL), (SEED_RESULT, SEED)] { if let Some(name) = result.getattr_opt(field)? { return Ok(Some((kind, name.extract()?))); @@ -102,147 +69,6 @@ fn kind_of(result: &Bound<'_, PyAny>) -> PyResult Ok(None) } -impl ConsumptionRecorder { - fn lock(&self) -> PyResult> { - self.recording - .lock() - .map_err(|_| PyRuntimeError::new_err("consumption recorder state is poisoned")) - } - - /// The succeeded rows of one execution that wrote its relation, with their schema. - fn succeeded(&self, key: &NodeKey) -> Option<(Location, Vec)> { - let (Some(node), Some(execution_id)) = (self.nodes.get(key), self.executions.get(key)) - else { - return None; - }; - let inputs = consumed_inputs(node, (&self.nodes, &self.history), &self.executions); - Some(( - Location::of(&node.relation), - succeeded_appends(node, (execution_id, &self.run_id), &inputs), - )) - } - - /// The recording with `key`'s succeeded rows queued. - fn queued(&self, mut recording: Recording, key: &NodeKey) -> Recording { - if let Some((location, rows)) = self.succeeded(key) { - recording.pending.entry(location).or_default().extend(rows); - recording.pending_executions += 1; - } - recording - } - - /// The recording after appending every pending row; a failure is reported, and its executions run again. - fn flushed(&self, py: Python<'_>, mut recording: Recording) -> Recording { - let pending = std::mem::take(&mut recording.pending); - let wrote: bool = !pending.is_empty(); - for (location, rows) in pending { - let created: bool = !recording.tables.insert(location.clone()); - let written: PyResult<()> = (if created { - Ok(()) - } else { - self.connection.ensure_table(py, &location) - }) - .and_then(|()| self.connection.append(py, &location, &rows)); - if let Err(error) = written { - recording.warnings.push(format!( - "could not record consumed inputs in schema {}: {error}; those models and seeds run again under --changed", - location.schema - )); - } - } - if wrote && let Err(error) = self.connection.close(py) { - recording.warnings.push(format!( - "could not close the consumed-input connection: {error}" - )); - } - recording.pending_executions = 0; - recording.last_flush = Instant::now(); - recording - } - - /// Queue the seeds whose error audits passed and flush everything. - pub(crate) fn finish( - &self, - py: Python<'_>, - build_result: &Bound<'_, PyAny>, - ) -> PyResult> { - let mut failed_audits: BTreeSet = BTreeSet::new(); - for audit in native_field(build_result, "source_audit_results")?.try_iter()? { - let audit: Bound<'_, PyAny> = audit?; - let target: Bound<'_, PyAny> = native_field(&audit, "attached_target_name")?; - if !target.is_none() - && enum_text(&native_field(&audit, "outcome")?)?.eq_ignore_ascii_case(ERROR_OUTCOME) - { - failed_audits.insert(target.extract()?); - } - } - let mut recording = self.lock()?; - for seed in std::mem::take(&mut recording.pending_seeds) { - if !failed_audits.contains(&seed.name) { - *recording = self.queued(std::mem::take(&mut *recording), &seed); - } - } - *recording = self.flushed(py, std::mem::take(&mut *recording)); - Ok(std::mem::take(&mut recording.warnings)) - } -} - -#[pymethods] -impl ConsumptionRecorder { - /// Forward a finished node, then queue a successful model, or a seed until its audits are known. - fn on_node_complete(&self, py: Python<'_>, node_result: Bound<'_, PyAny>) -> PyResult<()> { - self.base_on_node_complete.forward(&node_result)?; - let Some((kind, name)) = kind_of(&node_result)? else { - return Ok(()); - }; - if !enum_text(&native_field(&node_result, "status")?)?.eq_ignore_ascii_case(SUCCESS) { - return Ok(()); - } - let key = NodeKey { - kind: kind.to_owned(), - name, - }; - let mut recording = self.lock()?; - if kind == SEED { - recording.pending_seeds.push(key); - return Ok(()); - } - *recording = self.queued(std::mem::take(&mut *recording), &key); - if recording.pending_executions >= FLUSH_EXECUTIONS - || recording.last_flush.elapsed() >= FLUSH_INTERVAL - { - *recording = self.flushed(py, std::mem::take(&mut *recording)); - } - Ok(()) - } - - /// Fail a model or seed whose `started` row could not be appended, before it writes. - #[pyo3(signature = (*, entry, connection))] - fn before_node_write( - &self, - entry: Bound<'_, PyAny>, - connection: Bound<'_, PyAny>, - ) -> PyResult<()> { - let _ = connection; - let name: String = native_field(&entry, "name")?.extract()?; - let kind: &str = if entry.hasattr("csv_settings")? { - SEED - } else { - MODEL - }; - let key = NodeKey { - kind: kind.to_owned(), - name, - }; - match self.failed_start.get(&key) { - Some(error) => Err(PyRuntimeError::new_err(format!( - "not written: SQLBuild could not record that this {kind} started ({error}), so a crash could not be detected later" - ))), - None => Ok(()), - } - } -} - /// Schemas to read records from: all under `--changed`, else those of unselected upstreams. fn read_locations(facts: &BuildFacts, changed: bool) -> BTreeSet { let by_key: HashMap<&NodeKey, &NodeFacts> = @@ -267,13 +93,6 @@ fn read_locations(facts: &BuildFacts, changed: bool) -> BTreeSet { .collect() } -/// Everything the build needs from the recorder: the recorder itself and what `--changed` skipped. -pub(crate) struct Prepared<'py> { - pub(crate) recorder: Bound<'py, ConsumptionRecorder>, - pub(crate) decision: Option, - pub(crate) warnings: Vec, -} - /// Read records, decide `--changed` skips, mint execution IDs and append `started` rows. pub(crate) fn prepare<'py>( (plan, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/relation_reads.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs rename to crates/sqlbuild-python-planner/src/consumption/_helpers/relation_reads.rs diff --git a/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs b/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs new file mode 100644 index 0000000000..93339cad0d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs @@ -0,0 +1,17 @@ +//! The policy of a `--changed` request, or None without it; triggered jobs resolve on `project`. + +use crate::consumption::_helpers::policy; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_state::consumption::models::ChangedPolicy; + +/// The policy of a `--changed` request, or None without it; triggered jobs resolve on `project`. +#[inline] +pub fn changed_policy( + project_dir: &Bound<'_, PyAny>, + request: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, +) -> PyResult> { + policy::changed_policy(project_dir, request, project) +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/held_back_text.rs b/crates/sqlbuild-python-planner/src/consumption/main/held_back_text.rs new file mode 100644 index 0000000000..8dd3dee28a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/held_back_text.rs @@ -0,0 +1,10 @@ +//! Why `--changed` holds a node back, as one line for verbose output. + +use crate::consumption::_helpers::recorder; +use sqlbuild_state::consumption::models::HeldBack; + +/// Why `--changed` holds a node back, as one line for verbose output. +#[inline] +pub fn held_back_text(why: &HeldBack) -> String { + recorder::held_back_text(why) +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/kind_value.rs b/crates/sqlbuild-python-planner/src/consumption/main/kind_value.rs new file mode 100644 index 0000000000..a8e7e00e98 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/kind_value.rs @@ -0,0 +1,10 @@ +//! A change kind as `[changes] counts` and `--changed=` spell it. + +use crate::consumption::_helpers::policy; +use sqlbuild_state::consumption::models::ChangeKind; + +/// A change kind as `[changes] counts` and `--changed=` spell it. +#[inline] +pub fn kind_value(kind: ChangeKind) -> &'static str { + policy::kind_value(kind) +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/mod.rs b/crates/sqlbuild-python-planner/src/consumption/main/mod.rs new file mode 100644 index 0000000000..3f9152ccc5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/mod.rs @@ -0,0 +1,9 @@ +pub mod changed_policy; +pub mod held_back_text; +pub mod kind_value; +pub mod policy_line; +pub mod prepare; +pub mod preview_changed; +pub mod reach_value; +pub mod relation; +pub mod run_reason_text; diff --git a/crates/sqlbuild-python-planner/src/consumption/main/policy_line.rs b/crates/sqlbuild-python-planner/src/consumption/main/policy_line.rs new file mode 100644 index 0000000000..bcae7da848 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/policy_line.rs @@ -0,0 +1,10 @@ +//! One line saying what the policy counts and how far it reaches, as build and plan print it. + +use crate::consumption::_helpers::policy; +use sqlbuild_state::consumption::models::ChangedPolicy; + +/// One line saying what the policy counts and how far it reaches, as build and plan print it. +#[inline] +pub fn policy_line(policy: &ChangedPolicy) -> String { + policy::policy_line(policy) +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/prepare.rs b/crates/sqlbuild-python-planner/src/consumption/main/prepare.rs new file mode 100644 index 0000000000..ad53731267 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/prepare.rs @@ -0,0 +1,24 @@ +//! Read records, decide `--changed` skips, mint execution IDs and append `started` rows. + +use crate::consumption::_helpers::recorder; +use crate::consumption::models::Prepared; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_state::consumption::models::ChangedPolicy; + +/// Read records, decide `--changed` skips, mint execution IDs and append `started` rows. +#[inline] +pub fn prepare<'py>( + (plan, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter, connection_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (policy, full_refresh): (Option<&ChangedPolicy>, bool), + (run_id, base_on_node_complete): (String, Bound<'py, PyAny>), +) -> PyResult> { + recorder::prepare( + (plan, project), + (adapter, connection_config), + (policy, full_refresh), + (run_id, base_on_node_complete), + ) +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/preview_changed.rs b/crates/sqlbuild-python-planner/src/consumption/main/preview_changed.rs new file mode 100644 index 0000000000..256194047c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/preview_changed.rs @@ -0,0 +1,22 @@ +//! What `plan --changed` previews: what a build would run now, read without recording anything. + +use crate::consumption::_helpers::recorder; +use crate::consumption::models::ChangedDecision; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_state::consumption::models::ChangedPolicy; + +/// What `plan --changed` previews: what a build would run now, read without recording anything. +#[inline] +pub fn preview_changed( + (plan, project): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (policy, full_refresh): (&ChangedPolicy, bool), +) -> PyResult<(ChangedDecision, Vec)> { + recorder::preview_changed( + (plan, project), + (adapter, connection_config), + (policy, full_refresh), + ) +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/reach_value.rs b/crates/sqlbuild-python-planner/src/consumption/main/reach_value.rs new file mode 100644 index 0000000000..cc13d319ff --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/reach_value.rs @@ -0,0 +1,10 @@ +//! A reach as `[changes] reach` and `--reach` spell it. + +use crate::consumption::_helpers::policy; +use sqlbuild_state::consumption::models::Reach; + +/// A reach as `[changes] reach` and `--reach` spell it. +#[inline] +pub fn reach_value(reach: &Reach) -> &'static str { + policy::reach_value(reach) +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/relation.rs b/crates/sqlbuild-python-planner/src/consumption/main/relation.rs new file mode 100644 index 0000000000..8fc9e4aceb --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/relation.rs @@ -0,0 +1,13 @@ +//! The relation a model or seed destination names; a destination without a schema has an empty one. + +use crate::consumption::_helpers::relation_reads; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_state::consumption::models::RelationKey; + +/// The relation a model or seed destination names; a destination without a schema has an empty one. +#[inline] +pub fn relation(destination: &Bound<'_, PyAny>) -> PyResult { + relation_reads::relation(destination) +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/run_reason_text.rs b/crates/sqlbuild-python-planner/src/consumption/main/run_reason_text.rs new file mode 100644 index 0000000000..b6b2a62d8b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/run_reason_text.rs @@ -0,0 +1,10 @@ +//! Why `--changed` runs a node, as one line for verbose output. + +use crate::consumption::_helpers::recorder; +use sqlbuild_state::consumption::models::RunReason; + +/// Why `--changed` runs a node, as one line for verbose output. +#[inline] +pub fn run_reason_text(reason: &RunReason) -> String { + recorder::run_reason_text(reason) +} diff --git a/crates/sqlbuild-python-planner/src/consumption/mod.rs b/crates/sqlbuild-python-planner/src/consumption/mod.rs new file mode 100644 index 0000000000..a250790faf --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/mod.rs @@ -0,0 +1,5 @@ +//! What a build consumed: the `--changed` facts and policy, the consumption recorder and relation reads. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-planner/src/consumption/models.rs b/crates/sqlbuild-python-planner/src/consumption/models.rs new file mode 100644 index 0000000000..dbf9c58baf --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/models.rs @@ -0,0 +1,220 @@ +//! Public records of consumption. + +use crate::consumption::_helpers::facts::MODEL; +use crate::consumption::_helpers::facts::SEED; +use crate::consumption::_helpers::facts::enum_text; +use crate::consumption::_helpers::recorder::ERROR_OUTCOME; +use crate::consumption::_helpers::recorder::FLUSH_EXECUTIONS; +use crate::consumption::_helpers::recorder::FLUSH_INTERVAL; +use crate::consumption::_helpers::recorder::Recording; +use crate::consumption::_helpers::recorder::SUCCESS; +use crate::consumption::_helpers::recorder::kind_of; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::native_host::models::ConsumptionConnection; +use sqlbuild_python_front::native_host::models::Location; +use sqlbuild_python_front::native_host::models::NodeCompleteHook; +use sqlbuild_state::consumption::main::consumed_inputs::consumed_inputs; +use sqlbuild_state::consumption::main::succeeded_appends::succeeded_appends; +use sqlbuild_state::consumption::models::ChangedPolicy; +use sqlbuild_state::consumption::models::ConsumptionAppend; +use sqlbuild_state::consumption::models::ConsumptionHistory; +use sqlbuild_state::consumption::models::HeldBack; +use sqlbuild_state::consumption::models::NodeFacts; +use sqlbuild_state::consumption::models::NodeKey; +use sqlbuild_state::consumption::models::RunReason; +use std::collections::BTreeSet; +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::Instant; + +/// Appends `started` rows before a build and `succeeded` rows as its models and seeds complete. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct ConsumptionRecorder { + pub(crate) connection: ConsumptionConnection, + pub(crate) run_id: String, + pub(crate) nodes: HashMap, + pub(crate) history: ConsumptionHistory, + pub(crate) executions: HashMap, + pub(crate) failed_start: HashMap, + pub(crate) base_on_node_complete: NodeCompleteHook, + pub(crate) recording: Mutex, +} + +/// What `--changed` decided, for the build's report. +#[derive(Debug)] +pub struct ChangedDecision { + pub policy: ChangedPolicy, + pub run: usize, + /// Unchanged nodes, which are up to date. + pub skipped: Vec, + /// Nodes downstream of a change that wait for a later run. + pub held_back: Vec<(NodeKey, HeldBack)>, + pub reasons: Vec<(NodeKey, RunReason)>, +} + +impl ChangedDecision { + /// Every selected node `--changed` leaves unbuilt: unchanged and held back. + pub fn not_run(&self) -> impl Iterator { + self.skipped + .iter() + .chain(self.held_back.iter().map(|(key, _)| key)) + } +} + +impl ConsumptionRecorder { + pub(crate) fn lock(&self) -> PyResult> { + self.recording + .lock() + .map_err(|_| PyRuntimeError::new_err("consumption recorder state is poisoned")) + } + + /// The succeeded rows of one execution that wrote its relation, with their schema. + pub(crate) fn succeeded(&self, key: &NodeKey) -> Option<(Location, Vec)> { + let (Some(node), Some(execution_id)) = (self.nodes.get(key), self.executions.get(key)) + else { + return None; + }; + let inputs = consumed_inputs(node, (&self.nodes, &self.history), &self.executions); + Some(( + Location::of(&node.relation), + succeeded_appends(node, (execution_id, &self.run_id), &inputs), + )) + } + + /// The recording with `key`'s succeeded rows queued. + pub(crate) fn queued(&self, mut recording: Recording, key: &NodeKey) -> Recording { + if let Some((location, rows)) = self.succeeded(key) { + recording.pending.entry(location).or_default().extend(rows); + recording.pending_executions += 1; + } + recording + } + + /// The recording after appending every pending row; a failure is reported, and its executions run again. + pub(crate) fn flushed(&self, py: Python<'_>, mut recording: Recording) -> Recording { + let pending = std::mem::take(&mut recording.pending); + let wrote: bool = !pending.is_empty(); + for (location, rows) in pending { + let created: bool = !recording.tables.insert(location.clone()); + let written: PyResult<()> = (if created { + Ok(()) + } else { + self.connection.ensure_table(py, &location) + }) + .and_then(|()| self.connection.append(py, &location, &rows)); + if let Err(error) = written { + recording.warnings.push(format!( + "could not record consumed inputs in schema {}: {error}; those models and seeds run again under --changed", + location.schema + )); + } + } + if wrote && let Err(error) = self.connection.close(py) { + recording.warnings.push(format!( + "could not close the consumed-input connection: {error}" + )); + } + recording.pending_executions = 0; + recording.last_flush = Instant::now(); + recording + } + + /// Queue the seeds whose error audits passed and flush everything. + pub fn finish(&self, py: Python<'_>, build_result: &Bound<'_, PyAny>) -> PyResult> { + let mut failed_audits: BTreeSet = BTreeSet::new(); + for audit in native_field(build_result, "source_audit_results")?.try_iter()? { + let audit: Bound<'_, PyAny> = audit?; + let target: Bound<'_, PyAny> = native_field(&audit, "attached_target_name")?; + if !target.is_none() + && enum_text(&native_field(&audit, "outcome")?)?.eq_ignore_ascii_case(ERROR_OUTCOME) + { + failed_audits.insert(target.extract()?); + } + } + let mut recording = self.lock()?; + for seed in std::mem::take(&mut recording.pending_seeds) { + if !failed_audits.contains(&seed.name) { + *recording = self.queued(std::mem::take(&mut *recording), &seed); + } + } + *recording = self.flushed(py, std::mem::take(&mut *recording)); + Ok(std::mem::take(&mut recording.warnings)) + } +} + +#[pymethods] +impl ConsumptionRecorder { + /// Forward a finished node, then queue a successful model, or a seed until its audits are known. + pub(crate) fn on_node_complete( + &self, + py: Python<'_>, + node_result: Bound<'_, PyAny>, + ) -> PyResult<()> { + self.base_on_node_complete.forward(&node_result)?; + let Some((kind, name)) = kind_of(&node_result)? else { + return Ok(()); + }; + if !enum_text(&native_field(&node_result, "status")?)?.eq_ignore_ascii_case(SUCCESS) { + return Ok(()); + } + let key = NodeKey { + kind: kind.to_owned(), + name, + }; + let mut recording = self.lock()?; + if kind == SEED { + recording.pending_seeds.push(key); + return Ok(()); + } + *recording = self.queued(std::mem::take(&mut *recording), &key); + if recording.pending_executions >= FLUSH_EXECUTIONS + || recording.last_flush.elapsed() >= FLUSH_INTERVAL + { + *recording = self.flushed(py, std::mem::take(&mut *recording)); + } + Ok(()) + } + + /// Fail a model or seed whose `started` row could not be appended, before it writes. + #[pyo3(signature = (*, entry, connection))] + pub(crate) fn before_node_write( + &self, + entry: Bound<'_, PyAny>, + connection: Bound<'_, PyAny>, + ) -> PyResult<()> { + let _ = connection; + let name: String = native_field(&entry, "name")?.extract()?; + let kind: &str = if entry.hasattr("csv_settings")? { + SEED + } else { + MODEL + }; + let key = NodeKey { + kind: kind.to_owned(), + name, + }; + match self.failed_start.get(&key) { + Some(error) => Err(PyRuntimeError::new_err(format!( + "not written: SQLBuild could not record that this {kind} started ({error}), so a crash could not be detected later" + ))), + None => Ok(()), + } + } +} + +/// Everything the build needs from the recorder: the recorder itself and what `--changed` skipped. +#[derive(Debug)] +pub struct Prepared<'py> { + pub recorder: Bound<'py, ConsumptionRecorder>, + pub decision: Option, + pub warnings: Vec, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/exception_types.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/exception_types.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/freshness/exception_types.rs rename to crates/sqlbuild-python-planner/src/freshness/_helpers/exception_types.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs rename to crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs index a7712a43d3..9be12b1ec2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs @@ -1,5 +1,6 @@ //! Observe the current freshness of selected sources and compare it with previous state. +use crate::freshness::models::PreviousState; use std::collections::{HashMap, HashSet}; use pyo3::exceptions::{PyException, PyValueError}; @@ -12,10 +13,12 @@ use sqlbuild_core::freshness_text::main::freshness_json::freshness_json; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::freshness::observation::{ - Observer, observation_record, observation_source, observe_adapter_batch, observe_configured, - strategy, -}; +use crate::freshness::_helpers::observation::observation_record; +use crate::freshness::_helpers::observation::observation_source; +use crate::freshness::_helpers::observation::observe_adapter_batch; +use crate::freshness::_helpers::observation::observe_configured; +use crate::freshness::_helpers::observation::strategy; +use crate::freshness::models::Observer; use sqlbuild_core::freshness_text::main::freshness_status::freshness_status; use sqlbuild_core::freshness_text::models::{ FreshnessSourceResult, FreshnessStatus, StateComparison, @@ -34,11 +37,6 @@ const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavai const MISSING_STATE_MESSAGE: &str = "previous source freshness state missing"; const NOT_RETURNED_MESSAGE: &str = "freshness metadata was not returned"; -/// Previous state records by identity; absent means no comparison. -pub(crate) struct PreviousState<'py> { - pub(crate) by_identity: Option>, -} - fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { if value.is_none() { return Ok(None); diff --git a/crates/sqlbuild-python-planner/src/freshness/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/mod.rs new file mode 100644 index 0000000000..70eb32c83b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod exception_types; +pub(crate) mod freshness_observe; +pub(crate) mod observation; +pub(crate) mod planning; +pub(crate) mod publication; +pub(crate) mod state_io; +pub(crate) mod support; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs rename to crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs index 873f93551f..e3ebc77ef1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs @@ -1,5 +1,6 @@ //! Observe configured and adapter-metadata source freshness through adapter callbacks. +use crate::freshness::models::Observer; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDateTime, PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -7,10 +8,12 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use sqlbuild_state::freshness::main::single_value_problem::single_value_problem; use sqlbuild_state::freshness::models::QueryShape; -use crate::bindings::_helpers::freshness::support::{ - OBSERVATION_ERROR, exception_text, freshness_error, presumed_utc_datetime, presumed_utc_value, - text, -}; +use crate::freshness::_helpers::support::OBSERVATION_ERROR; +use crate::freshness::_helpers::support::exception_text; +use crate::freshness::_helpers::support::freshness_error; +use crate::freshness::_helpers::support::presumed_utc_datetime; +use crate::freshness::_helpers::support::presumed_utc_value; +use crate::freshness::_helpers::support::text; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -36,13 +39,6 @@ const DEFAULT_STRATEGY: &str = "sql"; const METADATA_OPERATION: &str = "source_freshness_metadata_observation"; const QUERY_OPERATION: &str = "source_freshness_query_observation"; -/// The adapter, connection and observation time every source is observed with. -pub(crate) struct Observer<'py> { - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) connection: Bound<'py, PyAny>, - pub(crate) observed_at: Bound<'py, PyAny>, -} - fn observation_error(py: Python<'_>, message: &str) -> pyo3::PyErr { freshness_error(py, (OBSERVATION_ERROR, message), None) } @@ -123,7 +119,7 @@ fn lifecycle<'py>( call_with(&host_entry(py, ExecutorHost::OperationLifecycle)?, &options) } -fn observation<'py>( +pub(crate) fn observation<'py>( source_name: &Bound<'py, PyAny>, (strategy, value_kind): (Bound<'py, PyAny>, Bound<'py, PyAny>), (data_version, observed_at): (Bound<'py, PyAny>, Bound<'py, PyAny>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs rename to crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs index 14b5a97b82..9c29d4f93d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs @@ -1,5 +1,6 @@ //! Planning-time source freshness: observe, compare with latest state and map downstream impact. +use crate::freshness::constants::PREVIEW_MESSAGE; use std::collections::{HashMap, HashSet}; use pyo3::prelude::{ @@ -13,13 +14,17 @@ use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_state::freshness::main::configuration_error::configuration_error; -use crate::bindings::_helpers::freshness::exception_types::SourceFreshnessObservationError; -use crate::bindings::_helpers::freshness::observation::{ - Observer, observation_record, observation_source, observe_adapter_batch, observe_configured, - strategy, unknown, -}; -use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; -use crate::bindings::_helpers::freshness::support::text; +use crate::freshness::_helpers::exception_types::SourceFreshnessObservationError; +use crate::freshness::_helpers::observation::observation_record; +use crate::freshness::_helpers::observation::observation_source; +use crate::freshness::_helpers::observation::observe_adapter_batch; +use crate::freshness::_helpers::observation::observe_configured; +use crate::freshness::_helpers::observation::strategy; +use crate::freshness::_helpers::observation::unknown; +use crate::freshness::_helpers::state_io::read_latest; +use crate::freshness::_helpers::support::text; +use crate::freshness::models::Observer; +use crate::freshness::models::StateRead; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; @@ -35,8 +40,6 @@ use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; const ADAPTER_STRATEGY: &str = "adapter"; const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavailable"; -/// Why a plan preview leaves a source's freshness unknown; its plan lists freshness as unknown. -pub(crate) const PREVIEW_MESSAGE: &str = "a plan preview does not observe source freshness"; /// The message of an adapter, freshness observation or preview access error, if it is one. fn handled_message<'py>(py: Python<'py>, error: &PyErr) -> PyResult>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/publication.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs rename to crates/sqlbuild-python-planner/src/freshness/_helpers/publication.rs index 0a3336c014..0c72f2cab2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/publication.rs @@ -5,7 +5,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::freshness::state_io::write_source_freshness_records; +use crate::freshness::_helpers::state_io::write_source_freshness_records; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs rename to crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs index 317e8a32e4..661bb0bdde 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs @@ -1,5 +1,6 @@ //! Source freshness state table DDL, latest-row reads and appends through adapter callbacks. +use crate::freshness::models::StateRead; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -11,9 +12,12 @@ use sqlbuild_state::freshness::constants::{ use sqlbuild_state::freshness::main::read_latest_sql::read_latest_sql; use sqlbuild_state::freshness::main::relevant_sources_sql::relevant_sources_sql; -use crate::bindings::_helpers::freshness::support::{ - INPUT_ERROR, exception_text, freshness_error, optional_str, presumed_utc_datetime, text, -}; +use crate::freshness::_helpers::support::INPUT_ERROR; +use crate::freshness::_helpers::support::exception_text; +use crate::freshness::_helpers::support::freshness_error; +use crate::freshness::_helpers::support::optional_str; +use crate::freshness::_helpers::support::presumed_utc_datetime; +use crate::freshness::_helpers::support::text; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -105,14 +109,6 @@ fn create_table_sql<'py>( Ok(text(py, &sql)) } -/// The callbacks and location one latest-state read uses. -pub(crate) struct StateRead<'py> { - pub(crate) connection: Bound<'py, PyAny>, - pub(crate) execute: Bound<'py, PyAny>, - pub(crate) render_qualified_name: Bound<'py, PyAny>, - pub(crate) render_read_latest_sql: Bound<'py, PyAny>, -} - fn empty_set<'py>(schema: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = schema.py(); new_record_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs rename to crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs index 93d3fb0850..dac1690a74 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs @@ -4,9 +4,8 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyString}; -use crate::bindings::_helpers::freshness::exception_types::{ - SourceFreshnessInputError, SourceFreshnessObservationError, -}; +use crate::freshness::_helpers::exception_types::SourceFreshnessInputError; +use crate::freshness::_helpers::exception_types::SourceFreshnessObservationError; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; use sqlbuild_python_front::state_rows::main::event_rows::date_time_typed::date_time_typed; diff --git a/crates/sqlbuild-python-planner/src/freshness/constants.rs b/crates/sqlbuild-python-planner/src/freshness/constants.rs new file mode 100644 index 0000000000..3c1731b37e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/constants.rs @@ -0,0 +1,4 @@ +//! Public constants of freshness. + +/// Why a plan preview leaves a source's freshness unknown; its plan lists freshness as unknown. +pub const PREVIEW_MESSAGE: &str = "a plan preview does not observe source freshness"; diff --git a/crates/sqlbuild-python-planner/src/freshness/main/append_freshness_records.rs b/crates/sqlbuild-python-planner/src/freshness/main/append_freshness_records.rs new file mode 100644 index 0000000000..eddd32404a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/append_freshness_records.rs @@ -0,0 +1,22 @@ +//! Append observed source freshness only after the affected selected models succeed. + +use crate::freshness::_helpers::publication; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Append observed source freshness only after the affected selected models succeed. +#[inline] +pub fn append_freshness_records<'py>( + (plan, successful_models): (&Bound<'py, PyAny>, &[String]), + adapter: &Bound<'py, PyAny>, + connection_config: &Bound<'py, PyAny>, + run_id: &Bound<'py, PyAny>, +) -> PyResult<()> { + publication::append_freshness_records( + (plan, successful_models), + adapter, + connection_config, + run_id, + ) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/build_direct_source_freshness_planning_result_function.rs b/crates/sqlbuild-python-planner/src/freshness/main/build_direct_source_freshness_planning_result_function.rs new file mode 100644 index 0000000000..0a2539f738 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/build_direct_source_freshness_planning_result_function.rs @@ -0,0 +1,10 @@ +//! `build_direct_source_freshness_planning_result` as a Python function. + +use crate::freshness::_helpers::planning::build_direct_source_freshness_planning_result; + +/// `build_direct_source_freshness_planning_result` as a Python function. +pub fn build_direct_source_freshness_planning_result_function( + py: pyo3::Python<'_>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_direct_source_freshness_planning_result, py) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/build_direct_source_freshness_propagation_result_function.rs b/crates/sqlbuild-python-planner/src/freshness/main/build_direct_source_freshness_propagation_result_function.rs new file mode 100644 index 0000000000..7ba669dcef --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/build_direct_source_freshness_propagation_result_function.rs @@ -0,0 +1,10 @@ +//! `build_direct_source_freshness_propagation_result` as a Python function. + +use crate::freshness::_helpers::planning::build_direct_source_freshness_propagation_result; + +/// `build_direct_source_freshness_propagation_result` as a Python function. +pub fn build_direct_source_freshness_propagation_result_function( + py: pyo3::Python<'_>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_direct_source_freshness_propagation_result, py) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/mod.rs b/crates/sqlbuild-python-planner/src/freshness/main/mod.rs new file mode 100644 index 0000000000..7fba126b1e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/mod.rs @@ -0,0 +1,15 @@ +pub mod append_freshness_records; +pub mod build_direct_source_freshness_planning_result_function; +pub mod build_direct_source_freshness_propagation_result_function; +pub mod normalize_observation; +pub mod observe_freshness; +pub mod presumed_utc_datetime; +pub mod read_latest; +pub mod register_exception_types; +pub mod register_freshness_observe; +pub mod register_observation; +pub mod register_planning; +pub mod register_publication; +pub mod register_state_io; +pub mod require_aware_utc_datetime; +pub mod row_record; diff --git a/crates/sqlbuild-python-planner/src/freshness/main/normalize_observation.rs b/crates/sqlbuild-python-planner/src/freshness/main/normalize_observation.rs new file mode 100644 index 0000000000..e06c17d3f3 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/normalize_observation.rs @@ -0,0 +1,13 @@ +//! Hold an observation's `observed_at` and datetime `data_version` as aware UTC values. + +use crate::freshness::_helpers::observation; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Hold an observation's `observed_at` and datetime `data_version` as aware UTC values. +#[inline] +pub fn normalize_observation(py: Python<'_>, values: Vec>) -> PyResult>> { + observation::normalize_observation(py, values) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/observe_freshness.rs b/crates/sqlbuild-python-planner/src/freshness/main/observe_freshness.rs new file mode 100644 index 0000000000..fee613d9f3 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/observe_freshness.rs @@ -0,0 +1,19 @@ +//! Observe the selected sources inside the run's observability scope. + +use crate::freshness::_helpers::freshness_observe; +use crate::freshness::models::Observer; +use crate::freshness::models::PreviousState; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_core::freshness_text::models::FreshnessSourceResult; + +/// Observe the selected sources inside the run's observability scope. +#[inline] +pub fn observe_freshness( + observer: &Observer<'_>, + (sources, select, exclude): (&Bound<'_, PyAny>, &[String], &[String]), + (run_id, state): (Option, &PreviousState<'_>), +) -> PyResult> { + freshness_observe::observe_freshness(observer, (sources, select, exclude), (run_id, state)) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/presumed_utc_datetime.rs b/crates/sqlbuild-python-planner/src/freshness/main/presumed_utc_datetime.rs new file mode 100644 index 0000000000..082387f417 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/presumed_utc_datetime.rs @@ -0,0 +1,12 @@ +//! A `datetime` in UTC, a naive or offset-less value read as UTC. + +use crate::freshness::_helpers::support; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A `datetime` in UTC, a naive or offset-less value read as UTC. +#[inline] +pub fn presumed_utc_datetime<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + support::presumed_utc_datetime(value) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/read_latest.rs b/crates/sqlbuild-python-planner/src/freshness/main/read_latest.rs new file mode 100644 index 0000000000..e2f079ba04 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/read_latest.rs @@ -0,0 +1,18 @@ +//! Latest records of one schema, trusting the caller-resolved table existence. + +use crate::freshness::_helpers::state_io; +use crate::freshness::models::StateRead; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Latest records of one schema, trusting the caller-resolved table existence. +#[inline] +pub fn read_latest<'py>( + read: &StateRead<'py>, + (database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + table_exists: bool, + source_names: Option<&[String]>, +) -> PyResult> { + state_io::read_latest(read, (database, schema), table_exists, source_names) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_exception_types.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_exception_types.rs new file mode 100644 index 0000000000..e6c1a66011 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_exception_types.rs @@ -0,0 +1,12 @@ +//! Register the exception types functions and classes. + +use crate::freshness::_helpers::exception_types; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the exception types functions and classes. +#[inline] +pub fn register_exception_types(module: &Bound<'_, PyModule>) -> PyResult<()> { + exception_types::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_freshness_observe.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_freshness_observe.rs new file mode 100644 index 0000000000..9a1da01e1b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_freshness_observe.rs @@ -0,0 +1,12 @@ +//! Register the freshness observe functions and classes. + +use crate::freshness::_helpers::freshness_observe; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the freshness observe functions and classes. +#[inline] +pub fn register_freshness_observe(module: &Bound<'_, PyModule>) -> PyResult<()> { + freshness_observe::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_observation.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_observation.rs new file mode 100644 index 0000000000..542b47c211 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_observation.rs @@ -0,0 +1,12 @@ +//! Register the observation functions and classes. + +use crate::freshness::_helpers::observation; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the observation functions and classes. +#[inline] +pub fn register_observation(module: &Bound<'_, PyModule>) -> PyResult<()> { + observation::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_planning.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_planning.rs new file mode 100644 index 0000000000..5e8252fbbc --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_planning.rs @@ -0,0 +1,12 @@ +//! Register the planning functions and classes. + +use crate::freshness::_helpers::planning; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the planning functions and classes. +#[inline] +pub fn register_planning(module: &Bound<'_, PyModule>) -> PyResult<()> { + planning::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_publication.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_publication.rs new file mode 100644 index 0000000000..0f2be945b7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_publication.rs @@ -0,0 +1,12 @@ +//! Register the publication functions and classes. + +use crate::freshness::_helpers::publication; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the publication functions and classes. +#[inline] +pub fn register_publication(module: &Bound<'_, PyModule>) -> PyResult<()> { + publication::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs new file mode 100644 index 0000000000..f9170c48cb --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs @@ -0,0 +1,12 @@ +//! Register the state io functions and classes. + +use crate::freshness::_helpers::state_io; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the state io functions and classes. +#[inline] +pub fn register_state_io(module: &Bound<'_, PyModule>) -> PyResult<()> { + state_io::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/require_aware_utc_datetime.rs b/crates/sqlbuild-python-planner/src/freshness/main/require_aware_utc_datetime.rs new file mode 100644 index 0000000000..d8c2df1d69 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/require_aware_utc_datetime.rs @@ -0,0 +1,15 @@ +//! An aware freshness `datetime` in UTC; a naive value is an observation error. + +use crate::freshness::_helpers::observation; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// An aware freshness `datetime` in UTC; a naive value is an observation error. +#[inline] +pub fn require_aware_utc_datetime<'py>( + value: &Bound<'py, PyAny>, + field_name: &str, +) -> PyResult> { + observation::require_aware_utc_datetime(value, field_name) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/row_record.rs b/crates/sqlbuild-python-planner/src/freshness/main/row_record.rs new file mode 100644 index 0000000000..829f08045b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/row_record.rs @@ -0,0 +1,12 @@ +//! Row record. + +use crate::freshness::_helpers::state_io; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Row record. +#[inline] +pub fn row_record<'py>(row: &Bound<'py, PyAny>) -> PyResult> { + state_io::row_record(row) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/mod.rs b/crates/sqlbuild-python-planner/src/freshness/mod.rs new file mode 100644 index 0000000000..f3bac990ff --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/mod.rs @@ -0,0 +1,6 @@ +//! Source freshness: observation, planning, publication and state, through adapter callbacks. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-planner/src/freshness/models.rs b/crates/sqlbuild-python-planner/src/freshness/models.rs new file mode 100644 index 0000000000..ba73434ddc --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/models.rs @@ -0,0 +1,27 @@ +//! Public records of freshness. + +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; + +/// Previous state records by identity; absent means no comparison. +#[derive(Debug)] +pub struct PreviousState<'py> { + pub by_identity: Option>, +} + +/// The adapter, connection and observation time every source is observed with. +#[derive(Debug)] +pub struct Observer<'py> { + pub adapter: Bound<'py, PyAny>, + pub connection: Bound<'py, PyAny>, + pub observed_at: Bound<'py, PyAny>, +} + +/// The callbacks and location one latest-state read uses. +#[derive(Debug)] +pub struct StateRead<'py> { + pub connection: Bound<'py, PyAny>, + pub execute: Bound<'py, PyAny>, + pub render_qualified_name: Bound<'py, PyAny>, + pub render_read_latest_sql: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python-planner/src/lib.rs b/crates/sqlbuild-python-planner/src/lib.rs new file mode 100644 index 0000000000..ee655109a6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/lib.rs @@ -0,0 +1,19 @@ +//! The plan layer of the `sqlbuild._native` bindings: execution plans and their entries, planner phases, changes, migrations, resolution, scenarios, strategy and warehouse state, freshness, consumption, microbatches, observations, lifecycle events and plan output. + +#![forbid(unsafe_code)] + +pub mod consumption; +pub mod freshness; +pub mod lifecycle_events; +pub mod microbatches; +pub mod observations; +pub mod plan_output; +pub mod planner_changes; +pub mod planner_entries; +pub mod planner_identity; +pub mod planner_migrations; +pub mod planner_phases; +pub mod planner_resolution; +pub mod planner_scenario; +pub mod planner_strategy; +pub mod planner_warehouse; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/catalog.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs rename to crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/catalog.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs rename to crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs index 5874ec0270..e1e7a6eb6d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs @@ -17,11 +17,13 @@ use sqlbuild_core::lifecycle_events::main::publishable_hook_name::publishable_ho use sqlbuild_core::lifecycle_events::main::run_bounds_problem::run_bounds_problem; use sqlbuild_core::lifecycle_events::models::{HookFacts, SequenceFact}; -use crate::bindings::_helpers::lifecycle_events::payloads::validation_error; -use crate::bindings::_helpers::lifecycle_events::validation::{ - frozen_json, known_event_check, optional_text_check, required_text_check, schema_version_check, - timestamp_check, -}; +use crate::lifecycle_events::_helpers::payloads::validation_error; +use crate::lifecycle_events::_helpers::validation::frozen_json; +use crate::lifecycle_events::_helpers::validation::known_event_check; +use crate::lifecycle_events::_helpers::validation::optional_text_check; +use crate::lifecycle_events::_helpers::validation::required_text_check; +use crate::lifecycle_events::_helpers::validation::schema_version_check; +use crate::lifecycle_events::_helpers::validation::timestamp_check; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/mod.rs new file mode 100644 index 0000000000..c387aad44f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod catalog; +pub(crate) mod envelopes; +pub(crate) mod payloads; +pub(crate) mod validation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/payloads.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs rename to crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/payloads.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs rename to crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs index 8aaa1062e8..1c32277205 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs @@ -6,9 +6,10 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::lifecycle_events::main::validate_known_event::validate_known_event; use sqlbuild_core::lifecycle_events::models::KnownEventFacts; -use crate::bindings::_helpers::lifecycle_events::payloads::{ - PathSegment, frozen, object_entries, validation_error, -}; +use crate::lifecycle_events::_helpers::payloads::PathSegment; +use crate::lifecycle_events::_helpers::payloads::frozen; +use crate::lifecycle_events::_helpers::payloads::object_entries; +use crate::lifecycle_events::_helpers::payloads::validation_error; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/main/mod.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/main/mod.rs new file mode 100644 index 0000000000..2aaf10eb01 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/main/mod.rs @@ -0,0 +1,3 @@ +pub mod register_catalog; +pub mod register_envelopes; +pub mod register_validation; diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_catalog.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_catalog.rs new file mode 100644 index 0000000000..9d6001c23a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_catalog.rs @@ -0,0 +1,12 @@ +//! Register the catalog functions and classes. + +use crate::lifecycle_events::_helpers::catalog; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the catalog functions and classes. +#[inline] +pub fn register_catalog(module: &Bound<'_, PyModule>) -> PyResult<()> { + catalog::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_envelopes.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_envelopes.rs new file mode 100644 index 0000000000..7c70eed1e1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_envelopes.rs @@ -0,0 +1,12 @@ +//! Register the envelopes functions and classes. + +use crate::lifecycle_events::_helpers::envelopes; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the envelopes functions and classes. +#[inline] +pub fn register_envelopes(module: &Bound<'_, PyModule>) -> PyResult<()> { + envelopes::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs new file mode 100644 index 0000000000..7717df24f4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs @@ -0,0 +1,12 @@ +//! Register the validation functions and classes. + +use crate::lifecycle_events::_helpers::validation; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the validation functions and classes. +#[inline] +pub fn register_validation(module: &Bound<'_, PyModule>) -> PyResult<()> { + validation::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/mod.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/mod.rs new file mode 100644 index 0000000000..e083d0b80f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/mod.rs @@ -0,0 +1,4 @@ +//! The lifecycle event catalogue, envelopes, payload values and validation. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs b/crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs rename to crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs diff --git a/crates/sqlbuild-python-planner/src/microbatches/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/microbatches/_helpers/mod.rs new file mode 100644 index 0000000000..dc529131de --- /dev/null +++ b/crates/sqlbuild-python-planner/src/microbatches/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod conversion; +pub(crate) mod projection; +pub(crate) mod statements; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs b/crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs rename to crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs index 1ebcc33d77..fc1e9012d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs +++ b/crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs @@ -12,9 +12,10 @@ use sqlbuild_state::microbatches::models::{ ReplayState, ReplayVersions, RequirementFact, TextInterval, }; -use crate::bindings::_helpers::microbatches::conversion::{ - BoundParser, enum_text, optional_text, timestamp, -}; +use crate::microbatches::_helpers::conversion::BoundParser; +use crate::microbatches::_helpers::conversion::enum_text; +use crate::microbatches::_helpers::conversion::optional_text; +use crate::microbatches::_helpers::conversion::timestamp; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs b/crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs rename to crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs index dee8a8be60..ed75d6dad8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs +++ b/crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs @@ -14,7 +14,8 @@ use sqlbuild_state::microbatches::models::{MicrobatchEventIdentity, ScopeRead}; use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; use sqlbuild_state::state_tables::models::TableColumn; -use crate::bindings::_helpers::microbatches::conversion::{optional_text, state_error}; +use crate::microbatches::_helpers::conversion::optional_text; +use crate::microbatches::_helpers::conversion::state_error; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::contracts::main::render_state_sql_literal::render_state_sql_literal; diff --git a/crates/sqlbuild-python-planner/src/microbatches/main/mod.rs b/crates/sqlbuild-python-planner/src/microbatches/main/mod.rs new file mode 100644 index 0000000000..a06eec5634 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/microbatches/main/mod.rs @@ -0,0 +1,3 @@ +pub mod register_projection; +pub mod register_statements; +pub mod timestamp; diff --git a/crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs b/crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs new file mode 100644 index 0000000000..4652b5c024 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs @@ -0,0 +1,12 @@ +//! Register the projection functions and classes. + +use crate::microbatches::_helpers::projection; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the projection functions and classes. +#[inline] +pub fn register_projection(module: &Bound<'_, PyModule>) -> PyResult<()> { + projection::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/microbatches/main/register_statements.rs b/crates/sqlbuild-python-planner/src/microbatches/main/register_statements.rs new file mode 100644 index 0000000000..42b4cab7d5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/microbatches/main/register_statements.rs @@ -0,0 +1,12 @@ +//! Register the statements functions and classes. + +use crate::microbatches::_helpers::statements; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the statements functions and classes. +#[inline] +pub fn register_statements(module: &Bound<'_, PyModule>) -> PyResult<()> { + statements::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/microbatches/main/timestamp.rs b/crates/sqlbuild-python-planner/src/microbatches/main/timestamp.rs new file mode 100644 index 0000000000..ceb639b4ac --- /dev/null +++ b/crates/sqlbuild-python-planner/src/microbatches/main/timestamp.rs @@ -0,0 +1,12 @@ +//! `value.timestamp()` of a `datetime`. + +use crate::microbatches::_helpers::conversion; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `value.timestamp()` of a `datetime`. +#[inline] +pub fn timestamp(value: &Bound<'_, PyAny>) -> PyResult { + conversion::timestamp(value) +} diff --git a/crates/sqlbuild-python-planner/src/microbatches/mod.rs b/crates/sqlbuild-python-planner/src/microbatches/mod.rs new file mode 100644 index 0000000000..7ba5cc3b54 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/microbatches/mod.rs @@ -0,0 +1,4 @@ +//! Microbatch history: events read from Python, their projections and the direct-mode state statements. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/mod.rs b/crates/sqlbuild-python-planner/src/observations/_helpers/mod.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/observations/mod.rs rename to crates/sqlbuild-python-planner/src/observations/_helpers/mod.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs b/crates/sqlbuild-python-planner/src/observations/_helpers/publication.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs rename to crates/sqlbuild-python-planner/src/observations/_helpers/publication.rs index 84d46233af..69b75275fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs +++ b/crates/sqlbuild-python-planner/src/observations/_helpers/publication.rs @@ -8,10 +8,11 @@ use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_quer use sqlbuild_analysis::planner_warehouse::models::CursorTagRequest; use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND}; -use crate::bindings::_helpers::consumption::relation_reads::relation; -use crate::bindings::_helpers::observations::source_capture::take_source_capture; -use crate::bindings::_helpers::planner_warehouse::constants::RUN_BOUNDED; -use crate::bindings::_helpers::planner_warehouse::cursors::{adapter_cursor_sql, bound_task}; +use crate::consumption::main::relation::relation; +use crate::observations::_helpers::source_capture::take_source_capture; +use crate::planner_warehouse::constants::RUN_BOUNDED; +use crate::planner_warehouse::main::inspection::adapter_cursor_sql::adapter_cursor_sql; +use crate::planner_warehouse::main::inspection::bound_task::bound_task; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs b/crates/sqlbuild-python-planner/src/observations/_helpers/source_capture.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs rename to crates/sqlbuild-python-planner/src/observations/_helpers/source_capture.rs diff --git a/crates/sqlbuild-python-planner/src/observations/main/append_build_observations.rs b/crates/sqlbuild-python-planner/src/observations/main/append_build_observations.rs new file mode 100644 index 0000000000..428c32de9c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/observations/main/append_build_observations.rs @@ -0,0 +1,19 @@ +//! Record successful incremental models' cursor `MAX` and the source columns planning described; failures only warn. + +use crate::observations::_helpers::publication; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; + +/// Record successful incremental models' cursor `MAX` and the source columns planning described; failures only warn. +#[inline] +pub fn append_build_observations( + (project, successful_models): (&Bound<'_, PyAny>, &[String]), + (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + run_id: &str, +) -> Vec { + publication::append_build_observations( + (project, successful_models), + (adapter, connection_config), + run_id, + ) +} diff --git a/crates/sqlbuild-python-planner/src/observations/main/arm_source_capture.rs b/crates/sqlbuild-python-planner/src/observations/main/arm_source_capture.rs new file mode 100644 index 0000000000..1f1448862a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/observations/main/arm_source_capture.rs @@ -0,0 +1,9 @@ +//! Start capturing described source columns; only a build arms it, so `sqb plan` records nothing. + +use crate::observations::_helpers::source_capture; + +/// Start capturing described source columns; only a build arms it, so `sqb plan` records nothing. +#[inline] +pub fn arm_source_capture() { + source_capture::arm_source_capture() +} diff --git a/crates/sqlbuild-python-planner/src/observations/main/capture_source_columns.rs b/crates/sqlbuild-python-planner/src/observations/main/capture_source_columns.rs new file mode 100644 index 0000000000..af1b51edac --- /dev/null +++ b/crates/sqlbuild-python-planner/src/observations/main/capture_source_columns.rs @@ -0,0 +1,9 @@ +//! Keep one source's described SQL hash and column names when a build armed the capture. + +use crate::observations::_helpers::source_capture; + +/// Keep one source's described SQL hash and column names when a build armed the capture. +#[inline] +pub fn capture_source_columns(source: &str, sql: &str, columns: &[String]) { + source_capture::capture_source_columns(source, sql, columns) +} diff --git a/crates/sqlbuild-python-planner/src/observations/main/mod.rs b/crates/sqlbuild-python-planner/src/observations/main/mod.rs new file mode 100644 index 0000000000..9919a39848 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/observations/main/mod.rs @@ -0,0 +1,4 @@ +pub mod append_build_observations; +pub mod arm_source_capture; +pub mod capture_source_columns; +pub mod take_source_capture; diff --git a/crates/sqlbuild-python-planner/src/observations/main/take_source_capture.rs b/crates/sqlbuild-python-planner/src/observations/main/take_source_capture.rs new file mode 100644 index 0000000000..e9d6c47289 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/observations/main/take_source_capture.rs @@ -0,0 +1,10 @@ +//! The captured source columns, disarming the capture. + +use crate::observations::_helpers::source_capture; +use crate::observations::_helpers::source_capture::DescribedSources; + +/// The captured source columns, disarming the capture. +#[inline] +pub fn take_source_capture() -> DescribedSources { + source_capture::take_source_capture() +} diff --git a/crates/sqlbuild-python-planner/src/observations/mod.rs b/crates/sqlbuild-python-planner/src/observations/mod.rs new file mode 100644 index 0000000000..c86c2d4322 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/observations/mod.rs @@ -0,0 +1,4 @@ +//! What a build observed, cursor maximums and described source columns, for plan previews. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs rename to crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs index bef1e1810a..c194f877f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs @@ -3,10 +3,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::plan_output::display::{ - aligned_timestamp_bounds, resolve_effective_microbatch_batch_size, -}; -use crate::bindings::_helpers::plan_output::values::{optional_text, text}; +use crate::plan_output::_helpers::display::aligned_timestamp_bounds; +use crate::plan_output::_helpers::display::resolve_effective_microbatch_batch_size; +use crate::plan_output::_helpers::values::optional_text; +use crate::plan_output::_helpers::values::text; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs rename to crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs index 4cd6d088fe..03633e50b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs @@ -16,7 +16,8 @@ use sqlbuild_core::plan_display::main::format_waiting_on_empty_inputs::format_wa use sqlbuild_core::plan_display::main::model_materialization_label::model_materialization_label as materialization_label; use sqlbuild_core::plan_display::models::{ModelLabelFacts, ParsedBound}; -use crate::bindings::_helpers::plan_output::values::{optional_text, text}; +use crate::plan_output::_helpers::values::optional_text; +use crate::plan_output::_helpers::values::text; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs rename to crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs index 606fc94820..1d35c1f3d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs @@ -1,9 +1,12 @@ //! Convert public migration models at the Python boundary for native presentation. -use crate::bindings::_helpers::plan_output::plan_text::display_limit; -use crate::bindings::_helpers::plan_output::values::{ - iso, items, optional_text, relation_name, text, texts, -}; +use crate::plan_output::_helpers::plan_text::display_limit; +use crate::plan_output::_helpers::values::iso; +use crate::plan_output::_helpers::values::items; +use crate::plan_output::_helpers::values::optional_text; +use crate::plan_output::_helpers::values::relation_name; +use crate::plan_output::_helpers::values::text; +use crate::plan_output::_helpers::values::texts; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cli_text::models::{ diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/mod.rs new file mode 100644 index 0000000000..3f6b3b4a7e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod cursor_details; +pub(crate) mod display; +pub(crate) mod migrations; +pub(crate) mod plan_json; +pub(crate) mod plan_text; +pub(crate) mod python_identity; +pub(crate) mod safety; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs rename to crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs index 983a0772dc..2dafc928e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs @@ -1,28 +1,35 @@ //! The JSON plan document, read from the planner's output by attribute. +use crate::plan_output::constants::CHANGED_KEY; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::plan_display::constants::SYNCED_COPY_KEY; -use crate::bindings::_helpers::plan_output::cursor_details::{ - CursorDetails, cursor_details, owner, -}; -use crate::bindings::_helpers::plan_output::plan_text::{model_config_diff, model_query_diff}; -use crate::bindings::_helpers::plan_output::python_identity::{ - definition_source_text, dependency_source_text, -}; -use crate::bindings::_helpers::plan_output::safety::{future_cursor_safety, maximum_start_safety}; -use crate::bindings::_helpers::plan_output::values::{ - any, dict, enum_value, iso, items, list, mapped, optional_enum_value, relation_name, text, -}; +use crate::plan_output::_helpers::cursor_details::CursorDetails; +use crate::plan_output::_helpers::cursor_details::cursor_details; +use crate::plan_output::_helpers::cursor_details::owner; +use crate::plan_output::_helpers::plan_text::model_config_diff; +use crate::plan_output::_helpers::plan_text::model_query_diff; +use crate::plan_output::_helpers::python_identity::definition_source_text; +use crate::plan_output::_helpers::python_identity::dependency_source_text; +use crate::plan_output::_helpers::safety::future_cursor_safety; +use crate::plan_output::_helpers::safety::maximum_start_safety; +use crate::plan_output::_helpers::values::any; +use crate::plan_output::_helpers::values::dict; +use crate::plan_output::_helpers::values::enum_value; +use crate::plan_output::_helpers::values::iso; +use crate::plan_output::_helpers::values::items; +use crate::plan_output::_helpers::values::list; +use crate::plan_output::_helpers::values::mapped; +use crate::plan_output::_helpers::values::optional_enum_value; +use crate::plan_output::_helpers::values::relation_name; +use crate::plan_output::_helpers::values::text; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::compile_outputs::main::indented_json::indented_json; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; -/// The plan JSON key of what `--changed` would run, hold back and skip. -pub(crate) const CHANGED_KEY: &str = "changed"; const MICROBATCH_MODE: &str = "microbatch"; const RENAMED_REASON: &str = "renamed"; const RENAMED_DECISION: &str = "renamed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs rename to crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index d950e7e477..546f456cdc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -12,19 +12,24 @@ use sqlbuild_core::cli_text::models::{ }; use sqlbuild_core::plan_display::main::format_query_diff::format_query_diff; -use crate::bindings::_helpers::plan_output::cursor_details::{ - CursorDetails, DEFERRED, RESOLVED, cursor_details, runtime_inputs_clause, -}; -use crate::bindings::_helpers::plan_output::display::{ - cursor_bound_display, format_no_input_rows, format_waiting_on_empty_inputs, - inclusive_cursor_end, model_materialization_label, -}; -use crate::bindings::_helpers::plan_output::migrations::pre_build_lines; -use crate::bindings::_helpers::plan_output::plan_json::unified_diff; -use crate::bindings::_helpers::plan_output::python_identity::{ - definition_source_text, dependency_source_text, -}; -use crate::bindings::_helpers::plan_output::values::{items, optional_text, text, texts}; +use crate::plan_output::_helpers::cursor_details::CursorDetails; +use crate::plan_output::_helpers::cursor_details::DEFERRED; +use crate::plan_output::_helpers::cursor_details::RESOLVED; +use crate::plan_output::_helpers::cursor_details::cursor_details; +use crate::plan_output::_helpers::cursor_details::runtime_inputs_clause; +use crate::plan_output::_helpers::display::cursor_bound_display; +use crate::plan_output::_helpers::display::format_no_input_rows; +use crate::plan_output::_helpers::display::format_waiting_on_empty_inputs; +use crate::plan_output::_helpers::display::inclusive_cursor_end; +use crate::plan_output::_helpers::display::model_materialization_label; +use crate::plan_output::_helpers::migrations::pre_build_lines; +use crate::plan_output::_helpers::plan_json::unified_diff; +use crate::plan_output::_helpers::python_identity::definition_source_text; +use crate::plan_output::_helpers::python_identity::dependency_source_text; +use crate::plan_output::_helpers::values::items; +use crate::plan_output::_helpers::values::optional_text; +use crate::plan_output::_helpers::values::text; +use crate::plan_output::_helpers::values::texts; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/python_identity.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs rename to crates/sqlbuild-python-planner/src/plan_output/_helpers/python_identity.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/safety.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs rename to crates/sqlbuild-python-planner/src/plan_output/_helpers/safety.rs index 62a673e840..5936609206 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/safety.rs @@ -4,7 +4,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::plan_output::values::{dict, enum_value, items}; +use crate::plan_output::_helpers::values::dict; +use crate::plan_output::_helpers::values::enum_value; +use crate::plan_output::_helpers::values::items; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs rename to crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs diff --git a/crates/sqlbuild-python-planner/src/plan_output/constants.rs b/crates/sqlbuild-python-planner/src/plan_output/constants.rs new file mode 100644 index 0000000000..058f1c37cd --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/constants.rs @@ -0,0 +1,4 @@ +//! Public constants of plan output. + +/// The plan JSON key of what `--changed` would run, hold back and skip. +pub const CHANGED_KEY: &str = "changed"; diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/aligned_timestamp_bounds.rs b/crates/sqlbuild-python-planner/src/plan_output/main/aligned_timestamp_bounds.rs new file mode 100644 index 0000000000..c4f123e3c5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/aligned_timestamp_bounds.rs @@ -0,0 +1,16 @@ +//! Comparable bounds, presuming a naive side UTC when the other is aware. + +use crate::plan_output::_helpers::display; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Comparable bounds, presuming a naive side UTC when the other is aware. +#[inline] +pub fn aligned_timestamp_bounds<'py>( + start: &Bound<'py, PyAny>, + end: &Bound<'py, PyAny>, +) -> PyResult> { + display::aligned_timestamp_bounds(start, end) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/any.rs b/crates/sqlbuild-python-planner/src/plan_output/main/any.rs new file mode 100644 index 0000000000..dcc624a50f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/any.rs @@ -0,0 +1,14 @@ +//! Any Rust value as a Python object. + +use crate::plan_output::_helpers::values; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Any Rust value as a Python object. +#[inline] +pub fn any<'py, T: IntoPyObjectExt<'py>>(py: Python<'py>, value: T) -> PyResult> { + values::any(py, value) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/cursor_bound_display.rs b/crates/sqlbuild-python-planner/src/plan_output/main/cursor_bound_display.rs new file mode 100644 index 0000000000..8576e6d79b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/cursor_bound_display.rs @@ -0,0 +1,16 @@ +//! A cursor bound for display, collapsing a whole-day midnight timestamp to a date. + +use crate::plan_output::_helpers::display; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A cursor bound for display, collapsing a whole-day midnight timestamp to a date. +#[inline] +pub fn cursor_bound_display( + value: &Bound<'_, PyAny>, + cursor_type: &Bound<'_, PyAny>, + cursor_grain: &Bound<'_, PyAny>, +) -> PyResult { + display::cursor_bound_display(value, cursor_type, cursor_grain) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/dict.rs b/crates/sqlbuild-python-planner/src/plan_output/main/dict.rs new file mode 100644 index 0000000000..2ae1bc859c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/dict.rs @@ -0,0 +1,17 @@ +//! A dictionary of the entries, in order. + +use crate::plan_output::_helpers::values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// A dictionary of the entries, in order. +#[inline] +pub fn dict<'py>( + py: Python<'py>, + entries: Vec<(&str, Bound<'py, PyAny>)>, +) -> PyResult> { + values::dict(py, entries) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/format_plan.rs b/crates/sqlbuild-python-planner/src/plan_output/main/format_plan.rs new file mode 100644 index 0000000000..f47ea03028 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/format_plan.rs @@ -0,0 +1,13 @@ +//! The text plan grouped by reason with inline detail, optionally closed by its completion line. + +use crate::plan_output::_helpers::plan_text; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// The text plan grouped by reason with inline detail, optionally closed by its completion line. +#[inline] +pub fn format_plan(py: Python<'_>, options: Option>) -> PyResult { + plan_text::format_plan(py, options) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/format_plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/main/format_plan_json.rs new file mode 100644 index 0000000000..90feb01094 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/format_plan_json.rs @@ -0,0 +1,13 @@ +//! The plan as one JSON document, `json.dumps(indent=2)`. + +use crate::plan_output::_helpers::plan_json; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// The plan as one JSON document, `json.dumps(indent=2)`. +#[inline] +pub fn format_plan_json(py: Python<'_>, options: Option>) -> PyResult { + plan_json::format_plan_json(py, options) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/future_cursor_safety.rs b/crates/sqlbuild-python-planner/src/plan_output/main/future_cursor_safety.rs new file mode 100644 index 0000000000..7dbf12ecf7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/future_cursor_safety.rs @@ -0,0 +1,12 @@ +//! Cap policy, bounds and physical input evidence of future-cursor safety, or `None`. + +use crate::plan_output::_helpers::safety; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Cap policy, bounds and physical input evidence of future-cursor safety, or `None`. +#[inline] +pub fn future_cursor_safety<'py>(evidence: &Bound<'py, PyAny>) -> PyResult> { + safety::future_cursor_safety(evidence) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/inclusive_cursor_end.rs b/crates/sqlbuild-python-planner/src/plan_output/main/inclusive_cursor_end.rs new file mode 100644 index 0000000000..359d9714ca --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/inclusive_cursor_end.rs @@ -0,0 +1,16 @@ +//! The last cursor value included by an exclusive end bound. + +use crate::plan_output::_helpers::display; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The last cursor value included by an exclusive end bound. +#[inline] +pub fn inclusive_cursor_end( + end: &Bound<'_, PyAny>, + cursor_type: &Bound<'_, PyAny>, + cursor_grain: &Bound<'_, PyAny>, +) -> PyResult { + display::inclusive_cursor_end(end, cursor_type, cursor_grain) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/items.rs b/crates/sqlbuild-python-planner/src/plan_output/main/items.rs new file mode 100644 index 0000000000..269caa6c1f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/items.rs @@ -0,0 +1,12 @@ +//! Every item of an iterable. + +use crate::plan_output::_helpers::values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Every item of an iterable. +#[inline] +pub fn items<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { + values::items(value) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/maximum_start_safety.rs b/crates/sqlbuild-python-planner/src/plan_output/main/maximum_start_safety.rs new file mode 100644 index 0000000000..c84e933338 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/maximum_start_safety.rs @@ -0,0 +1,12 @@ +//! Automatic-start policy and physical eligibility evidence, or `None`. + +use crate::plan_output::_helpers::safety; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Automatic-start policy and physical eligibility evidence, or `None`. +#[inline] +pub fn maximum_start_safety<'py>(evidence: &Bound<'py, PyAny>) -> PyResult> { + safety::maximum_start_safety(evidence) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/mod.rs b/crates/sqlbuild-python-planner/src/plan_output/main/mod.rs new file mode 100644 index 0000000000..1ffbc27d2f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/mod.rs @@ -0,0 +1,19 @@ +pub mod aligned_timestamp_bounds; +pub mod any; +pub mod cursor_bound_display; +pub mod dict; +pub mod format_plan; +pub mod format_plan_json; +pub mod future_cursor_safety; +pub mod inclusive_cursor_end; +pub mod items; +pub mod maximum_start_safety; +pub mod optional_facts; +pub mod register_display; +pub mod register_migrations; +pub mod register_plan_json; +pub mod register_plan_text; +pub mod register_safety; +pub mod resolve_bounded_cursor_override; +pub mod resolve_effective_microbatch_batch_size; +pub mod text; diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/optional_facts.rs b/crates/sqlbuild-python-planner/src/plan_output/main/optional_facts.rs new file mode 100644 index 0000000000..d5f5c7d1b4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/optional_facts.rs @@ -0,0 +1,13 @@ +//! Optional facts. + +use crate::plan_output::_helpers::display; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_core::plan_display::models::ModelLabelFacts; + +/// Optional facts. +#[inline] +pub fn optional_facts(entry: &Bound<'_, PyAny>) -> PyResult> { + display::optional_facts(entry) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs b/crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs new file mode 100644 index 0000000000..2971f3c847 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs @@ -0,0 +1,12 @@ +//! Register the display functions and classes. + +use crate::plan_output::_helpers::display; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the display functions and classes. +#[inline] +pub fn register_display(module: &Bound<'_, PyModule>) -> PyResult<()> { + display::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/register_migrations.rs b/crates/sqlbuild-python-planner/src/plan_output/main/register_migrations.rs new file mode 100644 index 0000000000..6f2e4c9382 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/register_migrations.rs @@ -0,0 +1,12 @@ +//! Register the migrations functions and classes. + +use crate::plan_output::_helpers::migrations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the migrations functions and classes. +#[inline] +pub fn register_migrations(module: &Bound<'_, PyModule>) -> PyResult<()> { + migrations::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_json.rs new file mode 100644 index 0000000000..a73f20e89e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_json.rs @@ -0,0 +1,12 @@ +//! Register the plan json functions and classes. + +use crate::plan_output::_helpers::plan_json; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the plan json functions and classes. +#[inline] +pub fn register_plan_json(module: &Bound<'_, PyModule>) -> PyResult<()> { + plan_json::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_text.rs new file mode 100644 index 0000000000..63c7475ed1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_text.rs @@ -0,0 +1,12 @@ +//! Register the plan text functions and classes. + +use crate::plan_output::_helpers::plan_text; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the plan text functions and classes. +#[inline] +pub fn register_plan_text(module: &Bound<'_, PyModule>) -> PyResult<()> { + plan_text::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/register_safety.rs b/crates/sqlbuild-python-planner/src/plan_output/main/register_safety.rs new file mode 100644 index 0000000000..9d82e7979d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/register_safety.rs @@ -0,0 +1,12 @@ +//! Register the safety functions and classes. + +use crate::plan_output::_helpers::safety; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the safety functions and classes. +#[inline] +pub fn register_safety(module: &Bound<'_, PyModule>) -> PyResult<()> { + safety::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/resolve_bounded_cursor_override.rs b/crates/sqlbuild-python-planner/src/plan_output/main/resolve_bounded_cursor_override.rs new file mode 100644 index 0000000000..3f045b15c1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/resolve_bounded_cursor_override.rs @@ -0,0 +1,22 @@ +//! A fully bounded inclusive operator override as an executable interval. + +use crate::plan_output::_helpers::display; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// A fully bounded inclusive operator override as an executable interval. +#[inline] +pub fn resolve_bounded_cursor_override<'py>( + start_cursor_override: &Bound<'py, PyAny>, + end_cursor_override: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, + cursor_grain: &Bound<'py, PyAny>, +) -> PyResult>> { + display::resolve_bounded_cursor_override( + start_cursor_override, + end_cursor_override, + cursor_type, + cursor_grain, + ) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/resolve_effective_microbatch_batch_size.rs b/crates/sqlbuild-python-planner/src/plan_output/main/resolve_effective_microbatch_batch_size.rs new file mode 100644 index 0000000000..aa542e4c8d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/resolve_effective_microbatch_batch_size.rs @@ -0,0 +1,15 @@ +//! The explicit effective batch-size token resolved against the replay grain. + +use crate::plan_output::_helpers::display; +use pyo3::prelude::PyErr; +use pyo3::prelude::Python; + +/// The explicit effective batch-size token resolved against the replay grain. +#[inline] +pub fn resolve_effective_microbatch_batch_size( + py: Python<'_>, + batch_size: &str, + effective_grain: &str, +) -> Result { + display::resolve_effective_microbatch_batch_size(py, batch_size, effective_grain) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/text.rs b/crates/sqlbuild-python-planner/src/plan_output/main/text.rs new file mode 100644 index 0000000000..fdc2e5954e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/text.rs @@ -0,0 +1,12 @@ +//! `str(value)`. + +use crate::plan_output::_helpers::values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `str(value)`. +#[inline] +pub fn text(value: &Bound<'_, PyAny>) -> PyResult { + values::text(value) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/mod.rs b/crates/sqlbuild-python-planner/src/plan_output/mod.rs new file mode 100644 index 0000000000..2560516c60 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/mod.rs @@ -0,0 +1,5 @@ +//! Plan output: the text and JSON plan documents, display fragments, cursor details and safety notes. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/actions.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs rename to crates/sqlbuild-python-planner/src/planner_changes/_helpers/actions.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs rename to crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs index f5a3ad7932..11fee40d95 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs @@ -1,9 +1,10 @@ //! Detect how each selected model and function changed since its recorded build. +use crate::planner_changes::models::ModelPass; use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::planner_changes::results::native_enum; +use crate::planner_changes::_helpers::results::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; @@ -31,14 +32,16 @@ use sqlbuild_analysis::planner_changes::models::{ SchemaFindingRow, }; -use crate::bindings::_helpers::planner_changes::renames::origin_names; -use crate::bindings::_helpers::planner_changes::results::{ - backfill_result, change_kind, planner_enum, replay_policy_error, schema_findings, -}; -use crate::bindings::_helpers::planner_identity::metadata::{ - compiled_function_fingerprint_sql, function_local_hashes, model_declared_columns_hash, - model_version_identity_metadata_json, -}; +use crate::planner_changes::_helpers::renames::origin_names; +use crate::planner_changes::_helpers::results::backfill_result; +use crate::planner_changes::_helpers::results::change_kind; +use crate::planner_changes::_helpers::results::planner_enum; +use crate::planner_changes::_helpers::results::replay_policy_error; +use crate::planner_changes::_helpers::results::schema_findings; +use crate::planner_identity::main::compiled_function_fingerprint_sql::compiled_function_fingerprint_sql; +use crate::planner_identity::main::function_local_hashes::function_local_hashes; +use crate::planner_identity::main::model_declared_columns_hash::model_declared_columns_hash; +use crate::planner_identity::main::model_version_identity_metadata_json::model_version_identity_metadata_json; use sqlbuild_project::types::InferredRow; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -60,15 +63,6 @@ use sqlbuild_python_project::semantic_shapes::main::snapshot_validity_columns::s const CHANGES_LOGGER: &str = "sqlbuild.planner.changes"; const REPLAY_CONFIG_KEY: &str = "replay_on_change"; -/// Inputs shared by every model of one detection pass. -pub(crate) struct ModelPass<'py> { - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) query_change_tracking: bool, - pub(crate) full_refresh: bool, - pub(crate) function_local_hashes: Bound<'py, PyAny>, - pub(crate) hook_version_hashes: Bound<'py, PyAny>, -} - /// Selected model and function changes. #[pyfunction(name = "detect_changes")] #[pyo3(signature = (**options))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs rename to crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs diff --git a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/mod.rs new file mode 100644 index 0000000000..2f2b6ad173 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod actions; +pub(crate) mod detect; +pub(crate) mod metadata; +pub(crate) mod renames; +pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs rename to crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/results.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs rename to crates/sqlbuild-python-planner/src/planner_changes/_helpers/results.rs diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs new file mode 100644 index 0000000000..c12d5b8a4d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs @@ -0,0 +1,16 @@ +//! Selected model and function changes against expected version hashes and metadata JSON. + +use crate::planner_changes::_helpers::detect; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Selected model and function changes against expected version hashes and metadata JSON. +#[inline] +pub fn detect_changes<'py>( + (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + full_refresh: bool, + expected: (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + detect::detect_changes((project, scope, snapshot), full_refresh, expected) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/detect_model_changes.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_model_changes.rs new file mode 100644 index 0000000000..8a9928cb1a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_model_changes.rs @@ -0,0 +1,17 @@ +//! One model's change and backfill. + +use crate::planner_changes::_helpers::detect; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// One model's change and backfill. +#[inline] +pub fn detect_model_changes<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + detect::detect_model_changes(py, options) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/mod.rs new file mode 100644 index 0000000000..d6db35a20a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/mod.rs @@ -0,0 +1,14 @@ +pub mod detect_changes; +pub mod detect_model_changes; +pub mod model_change; +pub mod native_enum; +pub mod plan_reference_renames; +pub mod planner_enum; +pub mod register_actions; +pub mod register_detect; +pub mod register_metadata; +pub mod register_renames; +pub mod replaces_incremental_relation; +pub mod resolve_model_actions; +pub mod scope_model_changes; +pub mod without_declared_columns_hash; diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/model_change.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/model_change.rs new file mode 100644 index 0000000000..37666185b0 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/model_change.rs @@ -0,0 +1,17 @@ +//! Model change. + +use crate::planner_changes::_helpers::detect; +use crate::planner_changes::models::ModelPass; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Model change. +#[inline] +pub fn model_change<'py>( + pass: &ModelPass<'py>, + model: &Bound<'py, PyAny>, + (expected_version_hash, expected_metadata_json): (Option, Option), +) -> PyResult> { + detect::model_change(pass, model, (expected_version_hash, expected_metadata_json)) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/native_enum.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/native_enum.rs new file mode 100644 index 0000000000..6f1ed3130d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/native_enum.rs @@ -0,0 +1,18 @@ +//! A member of a natively declared planner enumeration, from its value. + +use crate::planner_changes::_helpers::results; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_python_base::records::types::RecordId; + +/// A member of a natively declared planner enumeration, from its value. +#[inline] +pub fn native_enum<'py>( + py: Python<'py>, + name: RecordId, + value: &str, +) -> PyResult> { + results::native_enum(py, name, value) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/plan_reference_renames.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/plan_reference_renames.rs new file mode 100644 index 0000000000..4747e2cf64 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/plan_reference_renames.rs @@ -0,0 +1,17 @@ +//! This plan's renames plus recorded renames, read only when a selected query that references models changed. + +use crate::planner_changes::_helpers::renames; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// This plan's renames plus recorded renames, read only when a selected query that references models changed. +#[inline] +pub fn plan_reference_renames<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + renames::plan_reference_renames(py, options) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/planner_enum.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/planner_enum.rs new file mode 100644 index 0000000000..afdb8a753b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/planner_enum.rs @@ -0,0 +1,18 @@ +//! `TYPES_MODULE.(value)`. + +use crate::planner_changes::_helpers::results; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_python_base::records::types::RecordId; + +/// `TYPES_MODULE.(value)`. +#[inline] +pub fn planner_enum<'py>( + py: Python<'py>, + name: RecordId, + value: &str, +) -> PyResult> { + results::planner_enum(py, name, value) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/register_actions.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/register_actions.rs new file mode 100644 index 0000000000..20f5e62351 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/register_actions.rs @@ -0,0 +1,12 @@ +//! Register the actions functions and classes. + +use crate::planner_changes::_helpers::actions; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the actions functions and classes. +#[inline] +pub fn register_actions(module: &Bound<'_, PyModule>) -> PyResult<()> { + actions::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs new file mode 100644 index 0000000000..a2aa945ca0 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs @@ -0,0 +1,12 @@ +//! Register the detect functions and classes. + +use crate::planner_changes::_helpers::detect; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the detect functions and classes. +#[inline] +pub fn register_detect(module: &Bound<'_, PyModule>) -> PyResult<()> { + detect::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs new file mode 100644 index 0000000000..46b94c628a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs @@ -0,0 +1,12 @@ +//! Register the metadata functions and classes. + +use crate::planner_changes::_helpers::metadata; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the metadata functions and classes. +#[inline] +pub fn register_metadata(module: &Bound<'_, PyModule>) -> PyResult<()> { + metadata::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/register_renames.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/register_renames.rs new file mode 100644 index 0000000000..d008a65d5b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/register_renames.rs @@ -0,0 +1,12 @@ +//! Register the renames functions and classes. + +use crate::planner_changes::_helpers::renames; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the renames functions and classes. +#[inline] +pub fn register_renames(module: &Bound<'_, PyModule>) -> PyResult<()> { + renames::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/replaces_incremental_relation.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/replaces_incremental_relation.rs new file mode 100644 index 0000000000..373ae416f2 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/replaces_incremental_relation.rs @@ -0,0 +1,9 @@ +//! Incremental replacements, whose bounds must ignore the destination maximum. + +use crate::planner_changes::_helpers::actions; + +/// Incremental replacements, whose bounds must ignore the destination maximum. +#[inline] +pub fn replaces_incremental_relation(materialization_type: &str, action: &str) -> bool { + actions::replaces_incremental_relation(materialization_type, action) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/resolve_model_actions.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/resolve_model_actions.rs new file mode 100644 index 0000000000..94c44cdeaa --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/resolve_model_actions.rs @@ -0,0 +1,15 @@ +//! Each selected model's change with the backfill its own `replay_on_change` decided. + +use crate::planner_changes::_helpers::actions; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Each selected model's change with the backfill its own `replay_on_change` decided. +#[inline] +pub fn resolve_model_actions<'py>( + scope: &Bound<'py, PyAny>, + changes: &Bound<'py, PyAny>, +) -> PyResult> { + actions::resolve_model_actions(scope, changes) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/scope_model_changes.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/scope_model_changes.rs new file mode 100644 index 0000000000..b50f8bb6de --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/scope_model_changes.rs @@ -0,0 +1,21 @@ +//! Scope model changes. + +use crate::planner_changes::_helpers::detect; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Scope model changes. +#[inline] +pub fn scope_model_changes<'py>( + (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + query_change_tracking: Option, + (version_hashes, metadata_jsons): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + detect::scope_model_changes( + (project, scope, snapshot), + query_change_tracking, + (version_hashes, metadata_jsons), + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/without_declared_columns_hash.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/without_declared_columns_hash.rs new file mode 100644 index 0000000000..7bb7da6e35 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/without_declared_columns_hash.rs @@ -0,0 +1,9 @@ +//! Fingerprint metadata JSON without the declared-columns hash key, if present. + +use crate::planner_changes::_helpers::metadata; + +/// Fingerprint metadata JSON without the declared-columns hash key, if present. +#[inline] +pub fn without_declared_columns_hash(metadata_json: &str) -> String { + metadata::without_declared_columns_hash(metadata_json) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/mod.rs b/crates/sqlbuild-python-planner/src/planner_changes/mod.rs new file mode 100644 index 0000000000..0c35d95139 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/mod.rs @@ -0,0 +1,5 @@ +//! Planner change detection: how each model changed, its actions, renames and identity metadata. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-planner/src/planner_changes/models.rs b/crates/sqlbuild-python-planner/src/planner_changes/models.rs new file mode 100644 index 0000000000..ccd18e88d1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/models.rs @@ -0,0 +1,14 @@ +//! Public records of planner changes. + +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; + +/// Inputs shared by every model of one detection pass. +#[derive(Debug)] +pub struct ModelPass<'py> { + pub snapshot: Bound<'py, PyAny>, + pub query_change_tracking: bool, + pub full_refresh: bool, + pub function_local_hashes: Bound<'py, PyAny>, + pub hook_version_hashes: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/contexts.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/contexts.rs index f6b65deb03..135d35aeae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/contexts.rs @@ -4,9 +4,9 @@ use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; use pyo3::types::{PyDict, PyFrozenSet}; -use crate::bindings::_helpers::planner_entries::models::{ - ModelPlanContext, PlanEntryBuildInputs, PlannerRelationsContext, -}; +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_entries::models::PlanEntryBuildInputs; +use crate::planner_entries::models::PlannerRelationsContext; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::types::RecordId; @@ -121,7 +121,7 @@ impl ModelPlanContext { impl PlanEntryBuildInputs { #[new] #[pyo3(signature = (**kwargs))] - pub(crate) fn new(py: Python<'_>, kwargs: Option>) -> PyResult { + pub fn new(py: Python<'_>, kwargs: Option>) -> PyResult { let args = KeywordArguments::read( py, "PlanEntryBuildInputs", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/getters.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs similarity index 82% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/getters.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs index 11b359c3e3..589c6e3b93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/getters.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs @@ -3,29 +3,11 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::planner_entries::models::{ - ModelPlanContext, PlanEntryBuildInputs, PlannerRelationsContext, -}; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_entries::models::PlanEntryBuildInputs; +use crate::planner_entries::models::PlannerRelationsContext; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[ - GetterEntry { - class: class_of::, - read: read_model_plan_context, - }, - GetterEntry { - class: class_of::, - read: read_plan_entry_build_inputs, - }, - GetterEntry { - class: class_of::, - read: read_planner_relations_context, - }, -]; - -fn read_model_plan_context<'py>( +pub(crate) fn read_model_plan_context<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -55,7 +37,7 @@ fn read_model_plan_context<'py>( }) } -fn read_plan_entry_build_inputs<'py>( +pub(crate) fn read_plan_entry_build_inputs<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -87,7 +69,7 @@ fn read_plan_entry_build_inputs<'py>( }) } -fn read_planner_relations_context<'py>( +pub(crate) fn read_planner_relations_context<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/mod.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/mod.rs new file mode 100644 index 0000000000..8147db06b4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod contexts; +pub(crate) mod getters; +pub(crate) mod model_entries; +pub(crate) mod relations; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs index 91bedada89..907c9bfc8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs @@ -1,38 +1,44 @@ //! Per-model plan entries from resolved change results and warehouse snapshot facts. -use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; -use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; -use crate::bindings::_helpers::planner_entries::models::{ - ModelPlanContext, PlanEntriesRequest, PlanEntryBuildInputs, PlannerRelationsContext, -}; -use crate::bindings::_helpers::planner_entries::values::{ - MICROBATCH_CONCURRENCY_MESSAGE, error_of, native, none, planner_type, replaced, string, -}; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::full_refresh::{ - check_full_rebuild_allowed, describe_full_rebuild_cause, resolve_model_full_refresh, -}; -use crate::bindings::_helpers::planner_phases::models::ModelEntryResults; -use crate::bindings::_helpers::planner_resolution::cursor_ranges::{ - CursorWindowInputs, config_str, cursor_safety_configs, empty_cursor_inputs, microbatch_range, - model_cursor_overrides, plan_cursor_bounds, -}; -use crate::bindings::_helpers::planner_resolution::cursor_warnings::with_cursor_safety_warnings; -use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest as SqlRequest; -use crate::bindings::_helpers::planner_resolution::orchestration::{ - Resolution, model_hook_entries, resolve_model_sql, -}; -use crate::bindings::_helpers::planner_strategy::cursor_inputs::runtime_cursor_producer_names; -use crate::bindings::_helpers::planner_strategy::cursor_inputs::{ - build_cursor_input_relations, reject_unexposed_cursor_input_columns, -}; -use crate::bindings::_helpers::planner_strategy::lineage::reject_capped_watermark_inputs; -use crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_configured_microbatch_limits; -use crate::bindings::_helpers::planner_strategy::model_entries::assemble_model_plan_entry; -use crate::bindings::_helpers::planner_strategy::strategy::{ - materialization_of, resolve_model_plan_action, -}; -use crate::bindings::_helpers::planner_strategy::values::text; +use crate::planner_changes::main::detect_model_changes::detect_model_changes; +use crate::planner_changes::main::replaces_incremental_relation::replaces_incremental_relation; +use crate::planner_entries::_helpers::entries::values::MICROBATCH_CONCURRENCY_MESSAGE; +use crate::planner_entries::_helpers::entries::values::error_of; +use crate::planner_entries::_helpers::entries::values::native; +use crate::planner_entries::_helpers::entries::values::none; +use crate::planner_entries::_helpers::entries::values::planner_type; +use crate::planner_entries::_helpers::entries::values::replaced; +use crate::planner_entries::_helpers::entries::values::string; +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_entries::models::PlanEntriesRequest; +use crate::planner_entries::models::PlanEntryBuildInputs; +use crate::planner_entries::models::PlannerRelationsContext; +use crate::planner_phases::main::inputs::class_field::class_field; +use crate::planner_phases::main::rebuilds::check_full_rebuild_allowed::check_full_rebuild_allowed; +use crate::planner_phases::main::rebuilds::describe_full_rebuild_cause::describe_full_rebuild_cause; +use crate::planner_phases::main::rebuilds::resolve_model_full_refresh::resolve_model_full_refresh; +use crate::planner_phases::models::ModelEntryResults; +use crate::planner_resolution::main::cursor::config_str::config_str; +use crate::planner_resolution::main::cursor::cursor_safety_configs::cursor_safety_configs; +use crate::planner_resolution::main::cursor::empty_cursor_inputs::empty_cursor_inputs; +use crate::planner_resolution::main::cursor::microbatch_range::microbatch_range; +use crate::planner_resolution::main::cursor::model_cursor_overrides::model_cursor_overrides; +use crate::planner_resolution::main::cursor::plan_cursor_bounds::plan_cursor_bounds; +use crate::planner_resolution::main::cursor::with_cursor_safety_warnings::with_cursor_safety_warnings; +use crate::planner_resolution::main::references::model_hook_entries::model_hook_entries; +use crate::planner_resolution::main::references::resolve_model_sql::resolve_model_sql; +use crate::planner_resolution::models::CursorWindowInputs; +use crate::planner_resolution::models::ModelSqlRequest as SqlRequest; +use crate::planner_resolution::models::Resolution; +use crate::planner_strategy::main::cursor_inputs::build_cursor_input_relations::build_cursor_input_relations; +use crate::planner_strategy::main::cursor_inputs::reject_unexposed_cursor_input_columns::reject_unexposed_cursor_input_columns; +use crate::planner_strategy::main::cursor_inputs::runtime_cursor_producer_names::runtime_cursor_producer_names; +use crate::planner_strategy::main::lineage::reject_capped_watermark_inputs::reject_capped_watermark_inputs; +use crate::planner_strategy::main::microbatch_limits::apply_configured_microbatch_limits::apply_configured_microbatch_limits; +use crate::planner_strategy::main::model_entries::assemble_model_plan_entry::assemble_model_plan_entry; +use crate::planner_strategy::main::strategy::materialization_of::materialization_of; +use crate::planner_strategy::main::strategy::resolve_model_plan_action::resolve_model_plan_action; +use crate::planner_strategy::main::values::text::text; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -618,7 +624,7 @@ pub(crate) fn scope_overlaps( Ok(false) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(plan_model, module)?)?; module.add_function(wrap_pyfunction!(scope_overlaps, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs index efaf15b9fa..396627db3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs @@ -1,18 +1,17 @@ //! Relation locations and source metadata for plan entry construction. -use crate::bindings::_helpers::planner_entries::models::{ - PlannerRelationsContext, RelationsRequest, -}; -use crate::bindings::_helpers::planner_entries::values::planner_model; -use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; -use crate::bindings::_helpers::planner_resolution::locations::build_model_locations; -use crate::bindings::_helpers::planner_resolution::locations::build_seed_locations; -use crate::bindings::_helpers::planner_resolution::refs::apply_deferred_locations; -use crate::bindings::_helpers::planner_strategy::audit_entries::audit_is_selected; -use crate::bindings::_helpers::planner_warehouse::semantic_sources::semantic_source_columns; -use crate::bindings::_helpers::planner_warehouse::source_columns::gather_source_inspection; -use crate::bindings::_helpers::planner_warehouse::source_deferral::build_source_read_map; -use crate::bindings::_helpers::planner_warehouse::source_deferral::with_declared_source_reads; +use crate::planner_entries::_helpers::entries::values::planner_model; +use crate::planner_entries::models::PlannerRelationsContext; +use crate::planner_entries::models::RelationsRequest; +use crate::planner_resolution::main::references::apply_deferred_locations::apply_deferred_locations; +use crate::planner_resolution::main::references::build_function_locations::build_function_locations; +use crate::planner_resolution::main::references::build_model_locations::build_model_locations; +use crate::planner_resolution::main::references::build_seed_locations::build_seed_locations; +use crate::planner_strategy::main::audit_entries::audit_is_selected::audit_is_selected; +use crate::planner_warehouse::main::sources::build_source_read_map::build_source_read_map; +use crate::planner_warehouse::main::sources::gather_source_inspection::gather_source_inspection; +use crate::planner_warehouse::main::sources::semantic_source_columns::semantic_source_columns; +use crate::planner_warehouse::main::sources::with_declared_source_reads::with_declared_source_reads; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -251,6 +250,6 @@ fn build_planner_relations_context<'py>( }) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(build_planner_relations_context, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/values.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/values.rs index e8195c4f43..0e93c43efb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/values.rs @@ -11,11 +11,11 @@ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_key use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -pub(super) const MICROBATCH_CONCURRENCY_MESSAGE: &str = +pub(crate) const MICROBATCH_CONCURRENCY_MESSAGE: &str = "batch_concurrency > 1 requires concurrent microbatches; selected model(s): "; /// The native record `name(**pairs)`. -pub(super) fn native<'py>( +pub(crate) fn native<'py>( py: Python<'py>, name: RecordId, pairs: &[(&str, Bound<'py, PyAny>)], @@ -24,7 +24,7 @@ pub(super) fn native<'py>( } /// The planner record `name(**pairs)`. -pub(super) fn planner_model<'py>( +pub(crate) fn planner_model<'py>( py: Python<'py>, name: RecordId, pairs: &[(&str, Bound<'py, PyAny>)], @@ -33,7 +33,7 @@ pub(super) fn planner_model<'py>( } /// One member of a planner enumeration. -pub(super) fn planner_type<'py>( +pub(crate) fn planner_type<'py>( py: Python<'py>, class: RecordId, member: &str, @@ -42,22 +42,22 @@ pub(super) fn planner_type<'py>( } /// A Python string. -pub(super) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { +pub(crate) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { PyString::new(py, text).into_any() } /// A Python bool. -pub(super) fn boolean(py: Python<'_>, value: bool) -> PyResult> { +pub(crate) fn boolean(py: Python<'_>, value: bool) -> PyResult> { Ok(value.into_pyobject(py)?.to_owned().into_any()) } /// `None`. -pub(super) fn none(py: Python<'_>) -> Bound<'_, PyAny> { +pub(crate) fn none(py: Python<'_>) -> Bound<'_, PyAny> { py.None().into_bound(py) } /// `dataclasses.replace(object, **fields)`. -pub(super) fn replaced<'py>( +pub(crate) fn replaced<'py>( object: &Bound<'py, PyAny>, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { @@ -65,7 +65,7 @@ pub(super) fn replaced<'py>( } /// `class(message, **fields)` from `module`, as a raised error. -pub(super) fn error_of( +pub(crate) fn error_of( py: Python<'_>, class: ErrorId, message: &str, diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/mod.rs new file mode 100644 index 0000000000..62e4ffc8ef --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod entries; +pub(crate) mod plans; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs index d958342e66..241605c074 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs @@ -1,18 +1,21 @@ //! Clone planning: destination and origin entries, and deferred-clone graph traversal. -use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; -use crate::bindings::_helpers::planner_entries::values::{ - native, planner_model, planner_type, string, -}; -use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; -use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; -use crate::bindings::_helpers::planner_resolution::locations::build_model_locations; -use crate::bindings::_helpers::planner_resolution::locations::build_seed_locations; -use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest; -use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; -use crate::bindings::_helpers::planner_resolution::orchestration::{Resolution, resolve_model_sql}; -use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; -use crate::bindings::_helpers::planner_warehouse::source_columns::gather_source_columns; +use crate::planner_entries::_helpers::entries::values::native; +use crate::planner_entries::_helpers::entries::values::planner_model; +use crate::planner_entries::_helpers::entries::values::planner_type; +use crate::planner_entries::_helpers::entries::values::string; +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_entries::types::PhaseTimings; +use crate::planner_identity::main::build_compiled_function_fingerprint_sql::build_compiled_function_fingerprint_sql; +use crate::planner_resolution::main::references::build_function_locations::build_function_locations; +use crate::planner_resolution::main::references::build_model_locations::build_model_locations; +use crate::planner_resolution::main::references::build_seed_locations::build_seed_locations; +use crate::planner_resolution::main::references::resolve_function_sql::resolve_function_sql; +use crate::planner_resolution::main::references::resolve_model_sql::resolve_model_sql; +use crate::planner_resolution::models::ModelSqlRequest; +use crate::planner_resolution::models::Resolution; +use crate::planner_strategy::main::strategy::materialization_of::materialization_of; +use crate::planner_warehouse::main::sources::gather_source_columns::gather_source_columns; use pyo3::PyRef; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; @@ -53,9 +56,6 @@ const SOURCE_INSPECTION_PHASE: &str = "source_inspection"; /// The clone phase that plans entries from the compiled projects. const PLANNING_PHASE: &str = "planning"; -/// Each clone phase name with its seconds, in the order the phases ran. -pub(crate) type PhaseTimings = Vec<(String, f64)>; - /// Whether a model is explicitly disabled with a boolean `enabled = false`. fn disabled_model(model: &Bound<'_, PyAny>) -> PyResult { Ok(matches!( @@ -756,7 +756,7 @@ pub(crate) fn resolve_skipped_view_chain<'py>( CloneGraph::read(&args)?.walk(true) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(resolve_clone_boundary, module)?)?; module.add_function(wrap_pyfunction!(resolve_skipped_view_chain, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs index 2aa469ea8a..3d477ad341 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs @@ -1,18 +1,19 @@ //! Static plans for commands that do not build models, and direct seed identity reads. -use crate::bindings::_helpers::planner_entries::models::RelationsRequest; -use crate::bindings::_helpers::planner_entries::plan_output::{ - function_entry, selected_test_entries, test_selected, -}; -use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; -use crate::bindings::_helpers::planner_entries::values::{ - boolean, none, planner_model, planner_type, -}; -use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; -use crate::bindings::_helpers::planner_identity::metadata::build_seed_identity; -use crate::bindings::_helpers::planner_strategy::audit_entries::build_selected_audit_entries; -use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; -use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; +use crate::planner_entries::_helpers::entries::relations::planner_relations_context; +use crate::planner_entries::_helpers::entries::values::boolean; +use crate::planner_entries::_helpers::entries::values::none; +use crate::planner_entries::_helpers::entries::values::planner_model; +use crate::planner_entries::_helpers::entries::values::planner_type; +use crate::planner_entries::_helpers::plans::plan_output::function_entry; +use crate::planner_entries::_helpers::plans::plan_output::selected_test_entries; +use crate::planner_entries::_helpers::plans::plan_output::test_selected; +use crate::planner_entries::models::RelationsRequest; +use crate::planner_identity::main::build_compiled_function_fingerprint_sql::build_compiled_function_fingerprint_sql; +use crate::planner_identity::main::build_seed_identity::build_seed_identity; +use crate::planner_strategy::main::audit_entries::build_selected_audit_entries::build_selected_audit_entries; +use crate::planner_strategy::main::model_entries::extract_seed_columns::extract_seed_columns; +use crate::planner_strategy::main::strategy::materialization_of::materialization_of; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{ PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PySet, PySetMethods, PyString, @@ -514,7 +515,7 @@ pub(crate) fn read_selected_seed_fingerprints<'py>( Ok(fingerprints) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(resolve_static_command_scope, module)?)?; module.add_function(wrap_pyfunction!(resolve_static_relation_context, module)?)?; module.add_function(wrap_pyfunction!(build_audit_command_plan, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs index 0a182d7c85..119d103918 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs @@ -1,10 +1,13 @@ //! Display-only plans for external plan surfaces when warehouse planning cannot resolve. -use crate::bindings::_helpers::planner_entries::values::{ - boolean, native, none, planner_model, planner_type, string, -}; -use crate::bindings::_helpers::planner_phases::full_refresh::resolve_model_full_refresh; -use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; +use crate::planner_entries::_helpers::entries::values::boolean; +use crate::planner_entries::_helpers::entries::values::native; +use crate::planner_entries::_helpers::entries::values::none; +use crate::planner_entries::_helpers::entries::values::planner_model; +use crate::planner_entries::_helpers::entries::values::planner_type; +use crate::planner_entries::_helpers::entries::values::string; +use crate::planner_phases::main::rebuilds::resolve_model_full_refresh::resolve_model_full_refresh; +use crate::planner_strategy::main::strategy::materialization_of::materialization_of; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -173,6 +176,6 @@ pub(crate) fn build_display_only_sqlbuild_plan<'py>( ) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(build_display_only_sqlbuild_plan, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs similarity index 80% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs index 66962e8d0b..d7f6fcfd7d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs @@ -3,35 +3,36 @@ use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::planner_changes::actions::resolve_model_actions; -use crate::bindings::_helpers::planner_changes::detect::detect_changes; -use crate::bindings::_helpers::planner_entries::test_planning::{ - PlanningRequest, SqlTestPlanning, start_planning, -}; -use crate::bindings::_helpers::planner_entries::values::{native, replaced}; -use crate::bindings::_helpers::planner_phases::buildability::check_selected_scope_buildability; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::deferred_origins::defer_to_migration_origins; -use crate::bindings::_helpers::planner_phases::execution_changes::{ - build_planner_entry_results, reconcile_execution_changes, -}; -use crate::bindings::_helpers::planner_phases::identities::{ - build_planner_identity_context, detect_stale_warning_changes, -}; -use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; -use crate::bindings::_helpers::planner_phases::models::PlannerRuntime; -use crate::bindings::_helpers::planner_phases::models::{ - PlanOutputContext, PlanWarningContext, PlannedModelEntries, StoragePolicyContext, -}; -use crate::bindings::_helpers::planner_phases::output_assembly::unselected_upstream_models; -use crate::bindings::_helpers::planner_phases::output_assembly::{ - add_deferred_origin_notes, add_plan_metadata, add_plan_warnings, add_storage_policies, - base_plan_output, -}; -use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; -use crate::bindings::_helpers::planner_phases::warehouse_state::gather_planner_warehouse_state; -use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; -use crate::bindings::_helpers::planner_warehouse::source_freshness::build_planner_source_freshness_result; +use crate::planner_changes::main::detect_changes::detect_changes; +use crate::planner_changes::main::resolve_model_actions::resolve_model_actions; +use crate::planner_entries::_helpers::entries::values::native; +use crate::planner_entries::_helpers::entries::values::replaced; +use crate::planner_entries::_helpers::plans::test_planning::PlanningRequest; +use crate::planner_entries::_helpers::plans::test_planning::SqlTestPlanning; +use crate::planner_entries::_helpers::plans::test_planning::start_planning; +use crate::planner_phases::main::inputs::class_field::class_field; +use crate::planner_phases::main::inputs::gather_planner_warehouse_state::gather_planner_warehouse_state; +use crate::planner_phases::main::inputs::warehouse_field::warehouse_field; +use crate::planner_phases::main::outputs::add_deferred_origin_notes::add_deferred_origin_notes; +use crate::planner_phases::main::outputs::add_plan_metadata::add_plan_metadata; +use crate::planner_phases::main::outputs::add_plan_warnings::add_plan_warnings; +use crate::planner_phases::main::outputs::add_storage_policies::add_storage_policies; +use crate::planner_phases::main::outputs::base_plan_output::base_plan_output; +use crate::planner_phases::main::outputs::build_planner_identity_context::build_planner_identity_context; +use crate::planner_phases::main::outputs::check_selected_scope_buildability::check_selected_scope_buildability; +use crate::planner_phases::main::outputs::detect_stale_warning_changes::detect_stale_warning_changes; +use crate::planner_phases::main::outputs::unselected_upstream_models::unselected_upstream_models; +use crate::planner_phases::main::rebuilds::build_planner_entry_results::build_planner_entry_results; +use crate::planner_phases::main::rebuilds::defer_to_migration_origins::defer_to_migration_origins; +use crate::planner_phases::main::rebuilds::reconcile_execution_changes::reconcile_execution_changes; +use crate::planner_phases::models::PlanOutputContext; +use crate::planner_phases::models::PlanWarningContext; +use crate::planner_phases::models::PlannedModelEntries; +use crate::planner_phases::models::PlannerIdentityContext; +use crate::planner_phases::models::PlannerRuntime; +use crate::planner_phases::models::StoragePolicyContext; +use crate::planner_resolution::main::cursor::type_cursor_overrides::type_cursor_overrides; +use crate::planner_warehouse::main::sources::build_planner_source_freshness_result::build_planner_source_freshness_result; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; @@ -303,6 +304,6 @@ pub(crate) fn build_execution_plan<'py>( } /// Add the execution plan entry point. -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(build_execution_plan, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/mod.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/mod.rs new file mode 100644 index 0000000000..a7993b715c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod clone_plans; +pub(crate) mod command_plans; +pub(crate) mod display_plan; +pub(crate) mod execution_plan; +pub(crate) mod plan_output; +pub(crate) mod test_planning; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs index ee41bcdd48..254694485f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs @@ -1,13 +1,16 @@ //! `PlanOutput` assembly from planned model entries, seeds, sources, functions, audits and tests. -use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps; -use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; -use crate::bindings::_helpers::planner_entries::values::{ - error_of, native, none, planner_model, planner_type, string, -}; -use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; -use crate::bindings::_helpers::planner_strategy::audit_entries::build_selected_audit_entries; -use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; +use crate::planner_entries::_helpers::entries::model_entries::scope_overlaps; +use crate::planner_entries::_helpers::entries::values::error_of; +use crate::planner_entries::_helpers::entries::values::native; +use crate::planner_entries::_helpers::entries::values::none; +use crate::planner_entries::_helpers::entries::values::planner_model; +use crate::planner_entries::_helpers::entries::values::planner_type; +use crate::planner_entries::_helpers::entries::values::string; +use crate::planner_entries::models::PlanOutputInputs; +use crate::planner_resolution::main::references::resolve_function_sql::resolve_function_sql; +use crate::planner_strategy::main::audit_entries::build_selected_audit_entries::build_selected_audit_entries; +use crate::planner_strategy::main::model_entries::extract_seed_columns::extract_seed_columns; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -347,7 +350,7 @@ fn require_intermediate_loader_targets<'py>( } /// Build one canonical function entry from resolved relation inputs. -pub(super) fn function_entry<'py>( +pub(crate) fn function_entry<'py>( function: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, relations: &Bound<'py, PyAny>, @@ -463,7 +466,7 @@ fn function_entries<'py>( } /// Whether a test is named by the selection or covers a selected resource, and not excluded. -pub(super) fn test_selected( +pub(crate) fn test_selected( test: &Bound<'_, PyAny>, selected_keys: &Bound<'_, PyAny>, selection: &Bound<'_, PyAny>, @@ -482,13 +485,13 @@ pub(super) fn test_selected( } /// Planned SQL test entries and the warnings planning them produced. -pub(super) struct SelectedTests<'py> { +pub(crate) struct SelectedTests<'py> { pub(super) entries: Vec>, pub(super) warnings: Vec>, } /// Selected SQL test entries and their warnings, failing on invalid fixtures. -pub(super) fn selected_test_entries<'py>( +pub(crate) fn selected_test_entries<'py>( (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), selected_keys: &Bound<'py, PyAny>, selection: &Bound<'py, PyAny>, @@ -797,7 +800,7 @@ fn build_selected_test_entries<'py>( )) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(sql_test_model_chain_names_by_key_function(module)?)?; module.add_function(wrap_pyfunction!(build_selected_test_entries, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs index 083a50daba..ec3c28ce35 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs @@ -13,7 +13,7 @@ use pyo3::sync::PyOnceLock; use pyo3::types::PyDict; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; -use crate::bindings::_helpers::planner_entries::plan_output::plan_selected_sql_tests; +use crate::planner_entries::_helpers::plans::plan_output::plan_selected_sql_tests; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -257,7 +257,7 @@ fn start_sql_test_planning<'py>( } /// Add the planning handle and entry points, and join planning threads at interpreter exit. -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_function(wrap_pyfunction!(start_sql_test_planning, module)?)?; let hook = wrap_pyfunction!(join_sql_test_planning_threads, module)?; diff --git a/crates/sqlbuild-python-planner/src/planner_entries/constants.rs b/crates/sqlbuild-python-planner/src/planner_entries/constants.rs new file mode 100644 index 0000000000..c55b87786f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/constants.rs @@ -0,0 +1,26 @@ +//! Public constants of planner entries. + +use crate::planner_entries::_helpers::entries::getters::read_model_plan_context; +use crate::planner_entries::_helpers::entries::getters::read_plan_entry_build_inputs; +use crate::planner_entries::_helpers::entries::getters::read_planner_relations_context; +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_entries::models::PlanEntryBuildInputs; +use crate::planner_entries::models::PlannerRelationsContext; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_model_plan_context, + }, + GetterEntry { + class: class_of::, + read: read_plan_entry_build_inputs, + }, + GetterEntry { + class: class_of::, + read: read_planner_relations_context, + }, +]; diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/default_microbatch_limit_action.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/default_microbatch_limit_action.rs new file mode 100644 index 0000000000..ec25bb28d4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/default_microbatch_limit_action.rs @@ -0,0 +1,13 @@ +//! The microbatch limit action used when a project configures none. + +use crate::planner_entries::_helpers::entries::contexts; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The microbatch limit action used when a project configures none. +#[inline] +pub fn default_microbatch_limit_action(py: Python<'_>) -> PyResult> { + contexts::default_microbatch_limit_action(py) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/mod.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/mod.rs new file mode 100644 index 0000000000..86fd6222a7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/mod.rs @@ -0,0 +1,6 @@ +pub mod default_microbatch_limit_action; +pub mod plan_entries; +pub mod plan_model; +pub mod planner_relations_context; +pub mod register_model_entries; +pub mod register_relations; diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_entries.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_entries.rs new file mode 100644 index 0000000000..04fa332dc4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_entries.rs @@ -0,0 +1,12 @@ +//! Build model plan entries from snapshot and per-model resolved actions. + +use crate::planner_entries::_helpers::entries::model_entries; +use crate::planner_entries::models::PlanEntriesRequest; +use crate::planner_phases::models::ModelEntryResults; +use pyo3::prelude::PyResult; + +/// Build model plan entries from snapshot and per-model resolved actions. +#[inline] +pub fn plan_entries<'py>(request: PlanEntriesRequest<'py>) -> PyResult> { + model_entries::plan_entries(request) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_model.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_model.rs new file mode 100644 index 0000000000..586526494e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_model.rs @@ -0,0 +1,17 @@ +//! Detect changes and build a plan entry and warnings for a single model. + +use crate::planner_entries::_helpers::entries::model_entries; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// Detect changes and build a plan entry and warnings for a single model. +#[inline] +pub fn plan_model<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + model_entries::plan_model(py, kwargs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/planner_relations_context.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/planner_relations_context.rs new file mode 100644 index 0000000000..cb95c5607c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/planner_relations_context.rs @@ -0,0 +1,15 @@ +//! Resolve relation locations and source metadata for plan entry construction. + +use crate::planner_entries::_helpers::entries::relations; +use crate::planner_entries::models::RelationsRequest; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Resolve relation locations and source metadata for plan entry construction. +#[inline] +pub fn planner_relations_context<'py>( + request: RelationsRequest<'py>, +) -> PyResult> { + relations::planner_relations_context(request) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_model_entries.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_model_entries.rs new file mode 100644 index 0000000000..a72331b11a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_model_entries.rs @@ -0,0 +1,12 @@ +//! Register the model entries functions and classes. + +use crate::planner_entries::_helpers::entries::model_entries; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the model entries functions and classes. +#[inline] +pub fn register_model_entries(module: &Bound<'_, PyModule>) -> PyResult<()> { + model_entries::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_relations.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_relations.rs new file mode 100644 index 0000000000..8c37b936f1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_relations.rs @@ -0,0 +1,12 @@ +//! Register the relations functions and classes. + +use crate::planner_entries::_helpers::entries::relations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the relations functions and classes. +#[inline] +pub fn register_relations(module: &Bound<'_, PyModule>) -> PyResult<()> { + relations::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/mod.rs new file mode 100644 index 0000000000..81d411af3f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/mod.rs @@ -0,0 +1,2 @@ +pub mod entries; +pub mod plans; diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs new file mode 100644 index 0000000000..9f640a4d3c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs @@ -0,0 +1,17 @@ +//! Project selected audits from canonical static state. + +use crate::planner_entries::_helpers::plans::command_plans; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Project selected audits from canonical static state. +#[inline] +pub fn build_audit_command_plan<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, +) -> PyResult> { + command_plans::build_audit_command_plan(project, adapter, scope, relations) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_display_only_sqlbuild_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_display_only_sqlbuild_plan.rs new file mode 100644 index 0000000000..799a541dfc --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_display_only_sqlbuild_plan.rs @@ -0,0 +1,16 @@ +//! Build a display-only plan when warehouse planning cannot fully resolve. + +use crate::planner_entries::_helpers::plans::display_plan; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Build a display-only plan when warehouse planning cannot fully resolve. +#[inline] +pub fn build_display_only_sqlbuild_plan<'py>( + project: &Bound<'py, PyAny>, + selected_model_names: &Bound<'py, PyAny>, + full_refresh: bool, +) -> PyResult> { + display_plan::build_display_only_sqlbuild_plan(project, selected_model_names, full_refresh) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs new file mode 100644 index 0000000000..97c354073e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs @@ -0,0 +1,17 @@ +//! Build the execution plan for one selection against live warehouse state. + +use crate::planner_entries::_helpers::plans::execution_plan; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// Build the execution plan for one selection against live warehouse state. +#[inline] +pub fn build_execution_plan<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + execution_plan::build_execution_plan(py, kwargs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_relation_command_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_relation_command_plan.rs new file mode 100644 index 0000000000..67f9d38b94 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_relation_command_plan.rs @@ -0,0 +1,16 @@ +//! Project static relation data for Python checks. + +use crate::planner_entries::_helpers::plans::command_plans; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Project static relation data for Python checks. +#[inline] +pub fn build_relation_command_plan<'py>( + project: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, +) -> PyResult> { + command_plans::build_relation_command_plan(project, scope, relations) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_seed_command_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_seed_command_plan.rs new file mode 100644 index 0000000000..2620e8e657 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_seed_command_plan.rs @@ -0,0 +1,17 @@ +//! Project selected direct seed work and canonical identities. + +use crate::planner_entries::_helpers::plans::command_plans; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Project selected direct seed work and canonical identities. +#[inline] +pub fn build_seed_command_plan<'py>( + project: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, + fingerprints: Option>, +) -> PyResult> { + command_plans::build_seed_command_plan(project, scope, relations, fingerprints) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_test_command_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_test_command_plan.rs new file mode 100644 index 0000000000..4f87b69a22 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_test_command_plan.rs @@ -0,0 +1,17 @@ +//! Project selected SQL tests from canonical static state. + +use crate::planner_entries::_helpers::plans::command_plans; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// Project selected SQL tests from canonical static state. +#[inline] +pub fn build_test_command_plan<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + command_plans::build_test_command_plan(py, kwargs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/mod.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/mod.rs new file mode 100644 index 0000000000..cef1b11857 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/mod.rs @@ -0,0 +1,17 @@ +pub mod build_audit_command_plan; +pub mod build_display_only_sqlbuild_plan; +pub mod build_execution_plan; +pub mod build_relation_command_plan; +pub mod build_seed_command_plan; +pub mod build_test_command_plan; +pub mod plan_output_fields; +pub mod read_selected_seed_fingerprints; +pub mod register_clone_plans; +pub mod register_command_plans; +pub mod register_display_plan; +pub mod register_execution_plan; +pub mod register_plan_output; +pub mod register_test_planning; +pub mod resolve_static_command_scope; +pub mod resolve_static_relation_context; +pub mod timed_clone_planning; diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/plan_output_fields.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/plan_output_fields.rs new file mode 100644 index 0000000000..db4f0e1810 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/plan_output_fields.rs @@ -0,0 +1,15 @@ +//! The plan output's fields from planned entries and resolved relation inputs. + +use crate::planner_entries::_helpers::plans::plan_output; +use crate::planner_entries::models::PlanOutputInputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The plan output's fields from planned entries and resolved relation inputs. +#[inline] +pub fn plan_output_fields<'py>( + inputs: PlanOutputInputs<'py>, +) -> PyResult)>> { + plan_output::plan_output_fields(inputs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/read_selected_seed_fingerprints.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/read_selected_seed_fingerprints.rs new file mode 100644 index 0000000000..e98225ca3a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/read_selected_seed_fingerprints.rs @@ -0,0 +1,17 @@ +//! Read only fingerprint state required by selected direct seeds. + +use crate::planner_entries::_helpers::plans::command_plans; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Read only fingerprint state required by selected direct seeds. +#[inline] +pub fn read_selected_seed_fingerprints<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + seeds: &Bound<'py, PyAny>, +) -> PyResult> { + command_plans::read_selected_seed_fingerprints(adapter, connection, seeds) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_clone_plans.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_clone_plans.rs new file mode 100644 index 0000000000..9f4b261d29 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_clone_plans.rs @@ -0,0 +1,12 @@ +//! Register the clone plans functions and classes. + +use crate::planner_entries::_helpers::plans::clone_plans; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the clone plans functions and classes. +#[inline] +pub fn register_clone_plans(module: &Bound<'_, PyModule>) -> PyResult<()> { + clone_plans::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_command_plans.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_command_plans.rs new file mode 100644 index 0000000000..96f1bfd2f5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_command_plans.rs @@ -0,0 +1,12 @@ +//! Register the command plans functions and classes. + +use crate::planner_entries::_helpers::plans::command_plans; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the command plans functions and classes. +#[inline] +pub fn register_command_plans(module: &Bound<'_, PyModule>) -> PyResult<()> { + command_plans::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_display_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_display_plan.rs new file mode 100644 index 0000000000..c69fa8c119 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_display_plan.rs @@ -0,0 +1,12 @@ +//! Register the display plan functions and classes. + +use crate::planner_entries::_helpers::plans::display_plan; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the display plan functions and classes. +#[inline] +pub fn register_display_plan(module: &Bound<'_, PyModule>) -> PyResult<()> { + display_plan::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_execution_plan.rs new file mode 100644 index 0000000000..24db27cb5f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_execution_plan.rs @@ -0,0 +1,12 @@ +//! Add the execution plan entry point. + +use crate::planner_entries::_helpers::plans::execution_plan; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Add the execution plan entry point. +#[inline] +pub fn register_execution_plan(module: &Bound<'_, PyModule>) -> PyResult<()> { + execution_plan::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_plan_output.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_plan_output.rs new file mode 100644 index 0000000000..f589b81332 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_plan_output.rs @@ -0,0 +1,12 @@ +//! Register the plan output functions and classes. + +use crate::planner_entries::_helpers::plans::plan_output; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the plan output functions and classes. +#[inline] +pub fn register_plan_output(module: &Bound<'_, PyModule>) -> PyResult<()> { + plan_output::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_test_planning.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_test_planning.rs new file mode 100644 index 0000000000..4275aa0909 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_test_planning.rs @@ -0,0 +1,12 @@ +//! Add the planning handle and entry points, and join planning threads at interpreter exit. + +use crate::planner_entries::_helpers::plans::test_planning; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Add the planning handle and entry points, and join planning threads at interpreter exit. +#[inline] +pub fn register_test_planning(module: &Bound<'_, PyModule>) -> PyResult<()> { + test_planning::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_command_scope.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_command_scope.rs new file mode 100644 index 0000000000..9f2dd34476 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_command_scope.rs @@ -0,0 +1,16 @@ +//! Resolve command selection through the canonical build selector phase. + +use crate::planner_entries::_helpers::plans::command_plans; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Resolve command selection through the canonical build selector phase. +#[inline] +pub fn resolve_static_command_scope<'py>( + project: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, + auto_load_sources: bool, +) -> PyResult> { + command_plans::resolve_static_command_scope(project, selection, auto_load_sources) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs new file mode 100644 index 0000000000..f15397d452 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs @@ -0,0 +1,17 @@ +//! Resolve canonical locations and source routing without warehouse inventory. + +use crate::planner_entries::_helpers::plans::command_plans; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// Resolve canonical locations and source routing without warehouse inventory. +#[inline] +pub fn resolve_static_relation_context<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + command_plans::resolve_static_relation_context(py, kwargs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/timed_clone_planning.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/timed_clone_planning.rs new file mode 100644 index 0000000000..1e7f88937d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/timed_clone_planning.rs @@ -0,0 +1,18 @@ +//! Plan a clone, with the seconds spent inspecting sources and planning. + +use crate::planner_entries::_helpers::plans::clone_plans; +use crate::planner_entries::types::PhaseTimings; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// Plan a clone, with the seconds spent inspecting sources and planning. +#[inline] +pub fn timed_clone_planning<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult<(Bound<'py, PyTuple>, PhaseTimings)> { + clone_plans::timed_clone_planning(py, kwargs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/mod.rs b/crates/sqlbuild-python-planner/src/planner_entries/mod.rs new file mode 100644 index 0000000000..37203f0548 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/mod.rs @@ -0,0 +1,7 @@ +//! Plan entry and plan output assembly over native planner resolution. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-planner/src/planner_entries/models.rs b/crates/sqlbuild-python-planner/src/planner_entries/models.rs new file mode 100644 index 0000000000..cfd87635c9 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/models.rs @@ -0,0 +1,101 @@ +//! Public records of planner entries. + +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::pyclass; +use sqlbuild_python_project::native_host::models::ModelLookup; + +/// What resolving relation locations and source metadata for one scope reads. +#[derive(Debug)] +pub struct RelationsRequest<'py> { + pub project: Bound<'py, PyAny>, + pub adapter: Bound<'py, PyAny>, + pub connection: Bound<'py, PyAny>, + pub scope: Bound<'py, PyAny>, + pub deferral: Bound<'py, PyAny>, + pub project_config: Bound<'py, PyAny>, + pub local_config: Bound<'py, PyAny>, + pub known_source_columns: Bound<'py, PyAny>, + pub relation_keys: Bound<'py, PyAny>, +} + +/// What building model plan entries for one execution scope reads. +#[derive(Debug)] +pub struct PlanEntriesRequest<'py> { + pub project: Bound<'py, PyAny>, + pub adapter: Bound<'py, PyAny>, + pub scope: Bound<'py, PyAny>, + pub snapshot: Bound<'py, PyAny>, + pub relations: Bound<'py, PyAny>, + pub resolved_models: Bound<'py, PyAny>, + pub cursor_overrides: Bound<'py, PyAny>, + pub full_refresh: bool, + pub build_inputs: Bound<'py, PyAny>, + /// The effective worker limit execution will use. + pub concurrency: i64, +} + +/// What assembling the plan output's fields reads. +#[derive(Debug)] +pub struct PlanOutputInputs<'py> { + pub project: Bound<'py, PyAny>, + pub adapter: Bound<'py, PyAny>, + pub scope: Bound<'py, PyAny>, + pub snapshot: Bound<'py, PyAny>, + pub relations: Bound<'py, PyAny>, + pub changes: Bound<'py, PyAny>, + pub model_entries: Vec>, + pub model_warnings: Vec>, + pub reload_sources: Bound<'py, PyAny>, + pub extras: Bound<'py, PyAny>, +} + +/// Resolved relation and source inputs for plan entry construction. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct PlannerRelationsContext { + pub model_locations: Py, + pub seed_locations: Py, + pub function_locations: Py, + pub source_map: Py, + pub source_read_map: Py, + pub python_source_read_map: Py, + pub source_warehouse_columns: Py, + pub star_exclude_keyword: Py, + pub listed_source_names: Py, +} + +/// Relation lookups and source columns for building one model plan entry. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct ModelPlanContext { + pub model_locations: Py, + pub models_by_name: ModelLookup, + pub seed_locations: Py, + pub function_locations: Py, + pub source_map: Py, + pub source_warehouse_columns: Py, + pub star_exclude_keyword: Py, + pub functions_by_name: Py, + pub runtime_cursor_producer_names: Py, + pub future_cursor_config: Py, + pub start_cursor_config: Py, + pub invocation_time: Py, +} + +/// Blocked models and cursor overrides for plan entry building. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct PlanEntryBuildInputs { + pub source_freshness_blocked_model_names: Py, + pub external_blocked_model_names: Py, + pub start_cursor_override: Py, + pub end_cursor_override: Py, + pub future_cursor_config: Py, + pub start_cursor_config: Py, + pub invocation_time: Py, + pub max_microbatches: Py, + pub max_microbatches_is_override: bool, + pub microbatch_limit_action: Py, +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/types.rs b/crates/sqlbuild-python-planner/src/planner_entries/types.rs new file mode 100644 index 0000000000..c437576b63 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/types.rs @@ -0,0 +1,4 @@ +//! Public traits and type aliases of planner entries. + +/// Each clone phase name with its seconds, in the order the phases ran. +pub type PhaseTimings = Vec<(String, f64)>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs rename to crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs index 13e30cf38e..626150f3be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs @@ -15,9 +15,9 @@ use sqlbuild_analysis::planner_identity::models::{GraphKey, IdentityGraph, Ident use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::planner_identity::metadata::{ - function_local_hashes, model_version_identity_metadata_json, seed_identity, -}; +use crate::planner_identity::_helpers::metadata::function_local_hashes; +use crate::planner_identity::_helpers::metadata::model_version_identity_metadata_json; +use crate::planner_identity::_helpers::metadata::seed_identity; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs rename to crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs index f9a3ca7624..235e4a8444 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs @@ -16,7 +16,7 @@ use sqlbuild_analysis::planner_identity::main::function_fingerprint_sql::functio use sqlbuild_analysis::planner_identity::models::FunctionFingerprintFields; use sqlbuild_core::fingerprint_identity::main::stable_identity_hash::stable_identity_hash; -use crate::bindings::_helpers::microbatches::conversion::timestamp; +use crate::microbatches::main::timestamp::timestamp; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; diff --git a/crates/sqlbuild-python-planner/src/planner_identity/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/mod.rs new file mode 100644 index 0000000000..29511a7cd3 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod identities; +pub(crate) mod metadata; +pub(crate) mod staleness; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs rename to crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs index bbdd4ec732..69e575e0d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs @@ -1,6 +1,6 @@ //! Warnings for selected models stale through changed upstreams outside the run set. -use crate::bindings::_helpers::planner_changes::results::native_enum; +use crate::planner_changes::main::native_enum::native_enum; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use std::collections::{HashMap, HashSet}; @@ -17,7 +17,8 @@ use sqlbuild_analysis::planner_identity::models::{ GraphKey, StaleWarning, StalenessGraph, StalenessNames, }; -use crate::bindings::_helpers::planner_changes::detect::{ModelPass, model_change}; +use crate::planner_changes::main::model_change::model_change; +use crate::planner_changes::models::ModelPass; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/build_compiled_function_fingerprint_sql.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/build_compiled_function_fingerprint_sql.rs new file mode 100644 index 0000000000..c18cbfb4f0 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/build_compiled_function_fingerprint_sql.rs @@ -0,0 +1,12 @@ +//! Stable function definition text used for change fingerprints. + +use crate::planner_identity::_helpers::metadata; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Stable function definition text used for change fingerprints. +#[inline] +pub fn build_compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { + metadata::build_compiled_function_fingerprint_sql(function) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/build_function_local_hashes.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/build_function_local_hashes.rs new file mode 100644 index 0000000000..e44db01904 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/build_function_local_hashes.rs @@ -0,0 +1,16 @@ +//! Local-only semantic hashes of functions by name. + +use crate::planner_identity::_helpers::metadata; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Local-only semantic hashes of functions by name. +#[inline] +pub fn build_function_local_hashes<'py>( + functions: &Bound<'py, PyAny>, + dialect: Option<&str>, +) -> PyResult> { + metadata::build_function_local_hashes(functions, dialect) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/build_model_version_identity_metadata_json.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/build_model_version_identity_metadata_json.rs new file mode 100644 index 0000000000..aa308c9928 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/build_model_version_identity_metadata_json.rs @@ -0,0 +1,20 @@ +//! Deterministic non-query model fingerprint metadata JSON. + +use crate::planner_identity::_helpers::metadata; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Deterministic non-query model fingerprint metadata JSON. +#[inline] +pub fn build_model_version_identity_metadata_json<'py>( + model: &Bound<'py, PyAny>, + function_local_hashes: Option>, + hook_version_hashes: Option>, +) -> PyResult { + metadata::build_model_version_identity_metadata_json( + model, + function_local_hashes, + hook_version_hashes, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/build_seed_identity.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/build_seed_identity.rs new file mode 100644 index 0000000000..9269375d03 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/build_seed_identity.rs @@ -0,0 +1,12 @@ +//! `(identity_hash, metadata_json)` for one compiled seed. + +use crate::planner_identity::_helpers::metadata; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `(identity_hash, metadata_json)` for one compiled seed. +#[inline] +pub fn build_seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + metadata::build_seed_identity(seed) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/build_stale_out_of_selection_warnings.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/build_stale_out_of_selection_warnings.rs new file mode 100644 index 0000000000..71b9d7f830 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/build_stale_out_of_selection_warnings.rs @@ -0,0 +1,17 @@ +//! Warn for selected models stale through changed upstreams outside the run set. + +use crate::planner_identity::_helpers::staleness; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; + +/// Warn for selected models stale through changed upstreams outside the run set. +#[inline] +pub fn build_stale_out_of_selection_warnings<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + staleness::build_stale_out_of_selection_warnings(py, options) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/compiled_function_fingerprint_sql.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/compiled_function_fingerprint_sql.rs new file mode 100644 index 0000000000..22d68f5197 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/compiled_function_fingerprint_sql.rs @@ -0,0 +1,12 @@ +//! Stable function definition text used for change fingerprints. + +use crate::planner_identity::_helpers::metadata; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Stable function definition text used for change fingerprints. +#[inline] +pub fn compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { + metadata::compiled_function_fingerprint_sql(function) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/direct_model_version_identities.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/direct_model_version_identities.rs new file mode 100644 index 0000000000..e2df991bdf --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/direct_model_version_identities.rs @@ -0,0 +1,22 @@ +//! Direct model identities of `(functions, scope)`; unchanged nodes keep their `recorded` fingerprints. + +use crate::planner_identity::_helpers::identities; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Direct model identities of `(functions, scope)`; unchanged nodes keep their `recorded` fingerprints. +#[inline] +pub fn direct_model_version_identities<'py>( + (functions, scope): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + dialect: Option, + (seeds, source_hashes, hooks): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + recorded: &Bound<'py, PyAny>, +) -> PyResult> { + identities::direct_model_version_identities( + (functions, scope), + dialect, + (seeds, source_hashes, hooks), + recorded, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/function_local_hashes.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/function_local_hashes.rs new file mode 100644 index 0000000000..04f88888ea --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/function_local_hashes.rs @@ -0,0 +1,17 @@ +//! Local hashes of functions by name; unchanged functions keep the latest local hash `recorded` models saw. + +use crate::planner_identity::_helpers::metadata; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Local hashes of functions by name; unchanged functions keep the latest local hash `recorded` models saw. +#[inline] +pub fn function_local_hashes<'py>( + functions: &Bound<'py, PyAny>, + dialect: Option<&str>, + recorded: &Bound<'py, PyAny>, +) -> PyResult> { + metadata::function_local_hashes(functions, dialect, recorded) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/hook_version_hashes.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/hook_version_hashes.rs new file mode 100644 index 0000000000..49bb1068d4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/hook_version_hashes.rs @@ -0,0 +1,15 @@ +//! `{name: identity.version_hash}` of the discovered Python hooks. + +use crate::planner_identity::_helpers::identities; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// `{name: identity.version_hash}` of the discovered Python hooks. +#[inline] +pub fn hook_version_hashes<'py>( + hook_functions: &Bound<'py, PyAny>, +) -> PyResult> { + identities::hook_version_hashes(hook_functions) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/merge_recomputed_model_changes.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/merge_recomputed_model_changes.rs new file mode 100644 index 0000000000..e80012bf13 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/merge_recomputed_model_changes.rs @@ -0,0 +1,15 @@ +//! Resolved actions with recomputed model changes patched in where available. + +use crate::planner_identity::_helpers::identities; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Resolved actions with recomputed model changes patched in where available. +#[inline] +pub fn merge_recomputed_model_changes<'py>( + resolved_actions: &Bound<'py, PyAny>, + changes: &Bound<'py, PyAny>, +) -> PyResult> { + identities::merge_recomputed_model_changes(resolved_actions, changes) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/mod.rs new file mode 100644 index 0000000000..37ba3e304e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/mod.rs @@ -0,0 +1,18 @@ +pub mod build_compiled_function_fingerprint_sql; +pub mod build_function_local_hashes; +pub mod build_model_version_identity_metadata_json; +pub mod build_seed_identity; +pub mod build_stale_out_of_selection_warnings; +pub mod compiled_function_fingerprint_sql; +pub mod direct_model_version_identities; +pub mod function_local_hashes; +pub mod hook_version_hashes; +pub mod merge_recomputed_model_changes; +pub mod model_declared_columns_hash; +pub mod model_version_identity_metadata_json; +pub mod or_empty; +pub mod register_identities; +pub mod register_metadata; +pub mod register_staleness; +pub mod seed_identity; +pub mod with_honest_model_write_hashes; diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/model_declared_columns_hash.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/model_declared_columns_hash.rs new file mode 100644 index 0000000000..2953fb0967 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/model_declared_columns_hash.rs @@ -0,0 +1,12 @@ +//! The hash of a model's own declared columns, column families and contract setting. + +use crate::planner_identity::_helpers::metadata; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The hash of a model's own declared columns, column families and contract setting. +#[inline] +pub fn model_declared_columns_hash(model: &Bound<'_, PyAny>) -> PyResult { + metadata::model_declared_columns_hash(model) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/model_version_identity_metadata_json.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/model_version_identity_metadata_json.rs new file mode 100644 index 0000000000..2d04303772 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/model_version_identity_metadata_json.rs @@ -0,0 +1,20 @@ +//! Deterministic non-query model fingerprint metadata JSON. + +use crate::planner_identity::_helpers::metadata; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Deterministic non-query model fingerprint metadata JSON. +#[inline] +pub fn model_version_identity_metadata_json<'py>( + model: &Bound<'py, PyAny>, + function_local_hashes: Option<&Bound<'py, PyAny>>, + hook_version_hashes: Option<&Bound<'py, PyAny>>, +) -> PyResult { + metadata::model_version_identity_metadata_json( + model, + function_local_hashes, + hook_version_hashes, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/or_empty.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/or_empty.rs new file mode 100644 index 0000000000..16d965d69a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/or_empty.rs @@ -0,0 +1,16 @@ +//! `value or {}`. + +use crate::planner_identity::_helpers::metadata; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// `value or {}`. +#[inline] +pub fn or_empty<'py>( + value: Option>, + py: Python<'py>, +) -> PyResult> { + metadata::or_empty(value, py) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/register_identities.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/register_identities.rs new file mode 100644 index 0000000000..f4e7192563 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/register_identities.rs @@ -0,0 +1,12 @@ +//! Register the identities functions and classes. + +use crate::planner_identity::_helpers::identities; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the identities functions and classes. +#[inline] +pub fn register_identities(module: &Bound<'_, PyModule>) -> PyResult<()> { + identities::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/register_metadata.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/register_metadata.rs new file mode 100644 index 0000000000..3695c2f5b1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/register_metadata.rs @@ -0,0 +1,12 @@ +//! Register the metadata functions and classes. + +use crate::planner_identity::_helpers::metadata; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the metadata functions and classes. +#[inline] +pub fn register_metadata(module: &Bound<'_, PyModule>) -> PyResult<()> { + metadata::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/register_staleness.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/register_staleness.rs new file mode 100644 index 0000000000..41b0da1353 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/register_staleness.rs @@ -0,0 +1,12 @@ +//! Register the staleness functions and classes. + +use crate::planner_identity::_helpers::staleness; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the staleness functions and classes. +#[inline] +pub fn register_staleness(module: &Bound<'_, PyModule>) -> PyResult<()> { + staleness::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/seed_identity.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/seed_identity.rs new file mode 100644 index 0000000000..b0fd0ed69f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/seed_identity.rs @@ -0,0 +1,12 @@ +//! `(identity_hash, metadata_json)` of one compiled seed from its content and load-affecting config. + +use crate::planner_identity::_helpers::metadata; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `(identity_hash, metadata_json)` of one compiled seed from its content and load-affecting config. +#[inline] +pub fn seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + metadata::seed_identity(seed) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/with_honest_model_write_hashes.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/with_honest_model_write_hashes.rs new file mode 100644 index 0000000000..15e1a01620 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/with_honest_model_write_hashes.rs @@ -0,0 +1,17 @@ +//! Planned write hashes replaced by hashes based on upstreams available this run. + +use crate::planner_identity::_helpers::identities; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// Planned write hashes replaced by hashes based on upstreams available this run. +#[inline] +pub fn with_honest_model_write_hashes<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + identities::with_honest_model_write_hashes(py, options) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/mod.rs b/crates/sqlbuild-python-planner/src/planner_identity/mod.rs new file mode 100644 index 0000000000..cc3f2ebbb6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/mod.rs @@ -0,0 +1,4 @@ +//! Planner version identities, their metadata and hashes, and staleness warnings. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs index 6f8df1dd2e..423e54e922 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs @@ -1,12 +1,12 @@ //! Gather live candidate evidence without changing adapter inspection ownership. -use crate::bindings::_helpers::planner_migrations::column_objects::{ - config, definition_hash, lower, -}; -use crate::bindings::_helpers::planner_migrations::models::{ - ColumnCandidate, ColumnMigrationInputs, MigrationPhase, -}; -use crate::bindings::_helpers::planner_migrations::projections::parse_query_shape; +use crate::planner_migrations::_helpers::column::column_objects::config; +use crate::planner_migrations::_helpers::column::column_objects::definition_hash; +use crate::planner_migrations::_helpers::column::column_objects::lower; +use crate::planner_migrations::_helpers::projections::projections::parse_query_shape; +use crate::planner_migrations::models::ColumnCandidate; +use crate::planner_migrations::models::ColumnMigrationInputs; +use crate::planner_migrations::models::MigrationPhase; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; @@ -16,7 +16,7 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::q use sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; -pub(super) fn candidates<'py>( +pub(crate) fn candidates<'py>( (phase, columns): (&MigrationPhase<'py>, &ColumnMigrationInputs<'py>), dialect: &Bound<'py, PyAny>, ) -> PyResult>> { @@ -99,7 +99,7 @@ pub(super) fn candidates<'py>( Ok(result) } -pub(super) fn declared_names<'py>( +pub(crate) fn declared_names<'py>( candidate: &ColumnCandidate<'py>, ) -> PyResult> { let mut names: Vec = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs index 1ea1e3457b..f7307a2b22 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs @@ -2,13 +2,17 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::planner_migrations::column_candidates::declared_names; -use crate::bindings::_helpers::planner_migrations::column_objects::{lower, model, replace}; -use crate::bindings::_helpers::planner_migrations::columns::identical_renames; -use crate::bindings::_helpers::planner_migrations::columns::rename_hints; -use crate::bindings::_helpers::planner_migrations::constants::{AUTOMATIC, DONE}; -use crate::bindings::_helpers::planner_migrations::models::{CheckedColumns, ColumnCandidate}; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_migrations::_helpers::column::column_candidates::declared_names; +use crate::planner_migrations::_helpers::column::column_objects::lower; +use crate::planner_migrations::_helpers::column::column_objects::model; +use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_migrations::_helpers::column::columns::identical_renames; +use crate::planner_migrations::_helpers::column::columns::rename_hints; +use crate::planner_migrations::constants::AUTOMATIC; +use crate::planner_migrations::constants::DONE; +use crate::planner_migrations::models::CheckedColumns; +use crate::planner_migrations::models::ColumnCandidate; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; use sqlbuild_analysis::planner_migrations::main::column_decision::column_decision; @@ -24,7 +28,7 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_row use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::planner_phases::main::object::object; -pub(super) fn decide<'py>( +pub(crate) fn decide<'py>( runtime: &Bound<'py, PyAny>, candidate: &ColumnCandidate<'py>, events: &Bound<'py, PyAny>, @@ -148,7 +152,7 @@ pub(super) fn decide<'py>( Ok(entries) } -pub(super) fn checked<'py>( +pub(crate) fn checked<'py>( runtime: &Bound<'py, PyAny>, candidate: &ColumnCandidate<'py>, entries: Vec>, @@ -213,10 +217,12 @@ pub(super) fn checked<'py>( ], )?); } else { - warnings.push(super::column_warnings::unavailable( - &entry, - &reason.extract::()?, - )?); + warnings.push( + crate::planner_migrations::_helpers::column::column_warnings::unavailable( + &entry, + &reason.extract::()?, + )?, + ); } } Ok(CheckedColumns { @@ -225,7 +231,7 @@ pub(super) fn checked<'py>( }) } -pub(super) fn hints<'py>( +pub(crate) fn hints<'py>( candidate: &ColumnCandidate<'py>, entries: &[Bound<'py, PyAny>], ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs index 4c39e4bb25..1af84ca8fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs @@ -13,15 +13,15 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::q use sqlbuild_python_project::planner_identity::main::model_definition_hash::model_definition_hash; use sqlbuild_python_project::planner_phases::main::object::object; -pub(super) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { +pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() } -pub(super) fn lower(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn lower(value: &Bound<'_, PyAny>) -> PyResult { value.call_method0("lower")?.extract() } -pub(super) fn replace<'py>( +pub(crate) fn replace<'py>( value: &Bound<'py, PyAny>, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { @@ -32,7 +32,7 @@ pub(super) fn replace<'py>( replaced_keywords(value, &kwargs) } -pub(super) fn model<'py>( +pub(crate) fn model<'py>( py: Python<'py>, name: RecordId, fields: &[(&str, Bound<'py, PyAny>)], @@ -40,7 +40,7 @@ pub(super) fn model<'py>( new_record_with(py, name, fields) } -pub(super) fn native<'py>( +pub(crate) fn native<'py>( py: Python<'py>, name: RecordId, fields: &[(&str, Bound<'py, PyAny>)], @@ -48,13 +48,13 @@ pub(super) fn native<'py>( new_record_with(py, name, fields) } -pub(super) fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { +pub(crate) fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { let py = model.py(); Ok(config_text_object(py, &model_values(model)?, key)? .unwrap_or_else(|| py.None().into_bound(py))) } -pub(super) fn definition_hash<'py>( +pub(crate) fn definition_hash<'py>( model: &Bound<'py, PyAny>, snapshot: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs index f8302572dc..fbf5822472 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs @@ -1,13 +1,14 @@ //! Project column renames onto the existing snapshot and redirect cursor reads. -use crate::bindings::_helpers::planner_migrations::column_objects::{ - config, definition_hash, lower, native, replace, -}; -use crate::bindings::_helpers::planner_migrations::models::ColumnCandidate; -use crate::bindings::_helpers::planner_migrations::models::{ - ColumnMigrationInputs, MigrationPhase, -}; -use crate::bindings::_helpers::planner_migrations::projections::renames_explain_change; +use crate::planner_migrations::_helpers::column::column_objects::config; +use crate::planner_migrations::_helpers::column::column_objects::definition_hash; +use crate::planner_migrations::_helpers::column::column_objects::lower; +use crate::planner_migrations::_helpers::column::column_objects::native; +use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_migrations::_helpers::projections::projections::renames_explain_change; +use crate::planner_migrations::models::ColumnCandidate; +use crate::planner_migrations::models::ColumnMigrationInputs; +use crate::planner_migrations::models::MigrationPhase; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::imported::imported; @@ -22,7 +23,7 @@ use sqlbuild_python_project::native_host::models::ModelLookup; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; -pub(super) fn overlay<'py>( +pub(crate) fn overlay<'py>( snapshot: &Bound<'py, PyAny>, candidates: &[ColumnCandidate<'py>], entries: &[Bound<'py, PyAny>], @@ -129,7 +130,7 @@ pub(super) fn overlay<'py>( ) } -pub(super) fn cursors<'py>( +pub(crate) fn cursors<'py>( (phase, columns): (&MigrationPhase<'py>, &ColumnMigrationInputs<'py>), snapshot: Bound<'py, PyAny>, entries: &[Bound<'py, PyAny>], @@ -173,7 +174,7 @@ pub(super) fn cursors<'py>( }; targets.set_item(name, target)?; } - let redirected = super::planning_boundary::redirected_cursor_snapshots( + let redirected = crate::planner_migrations::_helpers::planning::planning_boundary::redirected_cursor_snapshots( phase, ( native_field(&snapshot, "existing_relations")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs similarity index 71% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs index 77ea8f59e0..8d94e97b16 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs @@ -1,11 +1,11 @@ //! Host column migration planning while Python owns warehouse inspection. -use crate::bindings::_helpers::planner_migrations::column_objects::replace; -use crate::bindings::_helpers::planner_migrations::known_input_columns::KnownInputColumns; -use crate::bindings::_helpers::planner_migrations::models::{ - ColumnMigrationInputs, ColumnMigrationPlanning, MigrationPhase, -}; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_migrations::_helpers::column::known_input_columns::KnownInputColumns; +use crate::planner_migrations::models::ColumnMigrationInputs; +use crate::planner_migrations::models::ColumnMigrationPlanning; +use crate::planner_migrations::models::MigrationPhase; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PySet, PySetMethods}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -67,7 +67,7 @@ pub(crate) fn plan_column_migrations<'py>( let runtime = phase.runtime.clone(); let snapshot = phase.snapshot.clone(); let adapter = runtime_field(&runtime, |runtime| &runtime.adapter)?; - let candidates = super::column_candidates::candidates( + let candidates = crate::planner_migrations::_helpers::column::column_candidates::candidates( (phase, columns), &adapter.call_method0("sql_analysis_dialect")?, )?; @@ -79,7 +79,9 @@ pub(crate) fn plan_column_migrations<'py>( warnings: Vec::new(), }); } - let state = super::state_inspection::inspection(&runtime, None)?; + let state = crate::planner_migrations::_helpers::discovery::state_inspection::inspection( + &runtime, None, + )?; let schemas = PySet::empty(py)?; for candidate in &candidates { let schema = native_field(&destination(&candidate.model)?, "schema")?; @@ -103,29 +105,50 @@ pub(crate) fn plan_column_migrations<'py>( let mut warnings: Vec> = Vec::new(); let mut hints: Vec> = Vec::new(); for candidate in &candidates { - let decided = super::column_entries::decide( + let decided = crate::planner_migrations::_helpers::column::column_entries::decide( &runtime, candidate, &native_field(&state, "column_events")?, &inputs, )?; let physical = mapping_get(&columns.physical_relations, name_object(&candidate.model)?)?; - let checked = super::column_entries::checked(&runtime, candidate, decided, &physical)?; - for hint in super::column_entries::hints(candidate, &checked.entries)?.try_iter()? { + let checked = crate::planner_migrations::_helpers::column::column_entries::checked( + &runtime, candidate, decided, &physical, + )?; + for hint in crate::planner_migrations::_helpers::column::column_entries::hints( + candidate, + &checked.entries, + )? + .try_iter()? + { hints.push(hint?); } entries.extend(checked.entries); warnings.extend(checked.warnings); } - let policy = super::planning_boundary::missing_origin_policy(&runtime)?; + let policy = + crate::planner_migrations::_helpers::planning::planning_boundary::missing_origin_policy( + &runtime, + )?; for entry in &mut entries { *entry = replace(entry, &[("missing_origin_policy", policy.clone())])?; - if let Some(warning) = super::column_warnings::entry_warning(entry)? { + if let Some(warning) = + crate::planner_migrations::_helpers::column::column_warnings::entry_warning(entry)? + { warnings.push(warning); } } - let overlaid = super::column_overlay::overlay(&snapshot, &candidates, &entries, &inputs)?; - let snapshot = super::column_overlay::cursors((phase, columns), overlaid, &entries)?; + let overlaid = crate::planner_migrations::_helpers::column::column_overlay::overlay( + &snapshot, + &candidates, + &entries, + &inputs, + )?; + let snapshot = crate::planner_migrations::_helpers::column::column_overlay::cursors( + (phase, columns), + overlaid, + &entries, + )?; Ok(ColumnMigrationPlanning { snapshot, entries, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs index ae22e7faae..865c9af4f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs @@ -1,11 +1,11 @@ //! Preserve column-migration diagnostic codes and public messages. -use crate::bindings::_helpers::planner_migrations::column_objects::native; -use crate::bindings::_helpers::planner_migrations::column_objects::text; -use crate::bindings::_helpers::planner_migrations::constants::{ - DONE, MISSING_ORIGIN_ALLOW_OUTCOME, MISSING_ORIGIN_CONFIRMATION_OUTCOME, - MISSING_ORIGIN_DENY_OUTCOME, -}; +use crate::planner_migrations::_helpers::column::column_objects::native; +use crate::planner_migrations::_helpers::column::column_objects::text; +use crate::planner_migrations::constants::DONE; +use crate::planner_migrations::constants::MISSING_ORIGIN_ALLOW_OUTCOME; +use crate::planner_migrations::constants::MISSING_ORIGIN_CONFIRMATION_OUTCOME; +use crate::planner_migrations::constants::MISSING_ORIGIN_DENY_OUTCOME; use pyo3::exceptions::PyKeyError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_migrations::main::column_warning::column_warning; @@ -14,7 +14,7 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -pub(super) fn warning<'py>( +pub(crate) fn warning<'py>( entry: &Bound<'py, PyAny>, code: &str, message: &str, @@ -40,7 +40,7 @@ pub(super) fn warning<'py>( ) } -pub(super) fn entry_warning<'py>(entry: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn entry_warning<'py>(entry: &Bound<'py, PyAny>) -> PyResult>> { let destination = native_field(entry, "destination")?; let qualified = native_field(&destination, "qualified_name")?; let relation: String = if qualified.is_truthy()? { @@ -94,7 +94,7 @@ pub(super) fn entry_warning<'py>(entry: &Bound<'py, PyAny>) -> PyResult( +pub(crate) fn unavailable<'py>( entry: &Bound<'py, PyAny>, reason: &str, ) -> PyResult> { @@ -117,7 +117,7 @@ pub(super) fn unavailable<'py>( } /// The build outcome a missing migration origin has under its policy. -pub(super) fn missing_origin_outcome(policy: &Bound<'_, PyAny>) -> PyResult<&'static str> { +pub(crate) fn missing_origin_outcome(policy: &Bound<'_, PyAny>) -> PyResult<&'static str> { match policy.extract::()?.as_str() { "allow" => Ok(MISSING_ORIGIN_ALLOW_OUTCOME), "require_confirmation" => Ok(MISSING_ORIGIN_CONFIRMATION_OUTCOME), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs index 7f1a140fdc..2f55f97b27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs @@ -70,7 +70,7 @@ pub(crate) fn identical_renames<'py>( for (old, new) in pairs { renames.set_item(old, new)?; } - super::projections::explain( + crate::planner_migrations::_helpers::projections::projections::explain( &previous, ¤t, &renames.into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/known_input_columns.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/known_input_columns.rs diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/mod.rs new file mode 100644 index 0000000000..e244c0e49d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod column_candidates; +pub(crate) mod column_entries; +pub(crate) mod column_objects; +pub(crate) mod column_overlay; +pub(crate) mod column_planning; +pub(crate) mod column_warnings; +pub(crate) mod columns; +pub(crate) mod known_input_columns; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery.rs similarity index 66% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery.rs index c9164a2d4e..d5fb030a4d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery.rs @@ -1,14 +1,13 @@ //! Native migration discovery over existing inspection and fingerprint cache boundaries. -use crate::bindings::_helpers::planner_migrations::matching::{ - match_migration_fingerprints, migration_destination_fingerprints, -}; -use crate::bindings::_helpers::planner_migrations::models::{ - DiscoveryRequest, FingerprintInputs, ModelMigrationDiscovery, -}; -use crate::bindings::_helpers::planner_migrations::{ - discovery_events, discovery_models, discovery_requests, -}; +use crate::planner_migrations::_helpers::discovery::discovery_events::renames; +use crate::planner_migrations::_helpers::discovery::discovery_models::ordered; +use crate::planner_migrations::_helpers::discovery::discovery_requests::automatic; +use crate::planner_migrations::_helpers::planning::matching::match_migration_fingerprints; +use crate::planner_migrations::_helpers::planning::matching::migration_destination_fingerprints; +use crate::planner_migrations::models::DiscoveryRequest; +use crate::planner_migrations::models::FingerprintInputs; +use crate::planner_migrations::models::ModelMigrationDiscovery; use pyo3::prelude::{PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple, PyTupleMethods}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -17,7 +16,7 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::name_object: use std::collections::BTreeSet; /// Manual and automatically discovered migration requests with their destination fingerprints. -pub(super) fn discover_model_migrations<'py>( +pub(crate) fn discover_model_migrations<'py>( request: DiscoveryRequest<'py>, ) -> PyResult> { let DiscoveryRequest { @@ -31,12 +30,17 @@ pub(super) fn discover_model_migrations<'py>( fingerprints: cache, } = request; let py = runtime.py(); - let models = discovery_models::ordered(&runtime)?; + let models = ordered(&runtime)?; let declared_names = PyDict::new(py); for declaration in &declarations { declared_names.set_item(&declaration.model_name, py.None())?; } - let triggered = discovery_models::triggered(&models, &scope, &snapshot, &declared_names)?; + let triggered = crate::planner_migrations::_helpers::discovery::discovery_models::triggered( + &models, + &scope, + &snapshot, + &declared_names, + )?; if !triggered && manual.is_empty() { return Ok(ModelMigrationDiscovery { requests: Vec::new(), @@ -44,7 +48,9 @@ pub(super) fn discover_model_migrations<'py>( destination_fingerprints: PyDict::new(py), }); } - let metadata = discovery_models::metadata(&runtime, &models)?; + let metadata = crate::planner_migrations::_helpers::discovery::discovery_models::metadata( + &runtime, &models, + )?; let inputs = FingerprintInputs { metadata_jsons: metadata.into_any(), dialect: native_field(&runtime, "adapter")?.call_method0("sql_analysis_dialect")?, @@ -54,12 +60,15 @@ pub(super) fn discover_model_migrations<'py>( &native_field(&state, "inspect_schemas")?, &[("schemas", project_schemas)], )?; - let declared = discovery_events::declared(&declarations, &state)?; + let declared = crate::planner_migrations::_helpers::discovery::discovery_events::declared( + &declarations, + &state, + )?; let names = PyDict::new(py); for item in &models { names.set_item(name_object(item)?, item)?; } - let event_renames = discovery_events::renames(&runtime, &names, &state)?; + let event_renames = renames(&runtime, &names, &state)?; for (name, origin) in declared.iter() { event_renames.set_item(name, origin)?; } @@ -86,9 +95,16 @@ pub(super) fn discover_model_migrations<'py>( unbuilt.set_item(name, item)?; } } - let resumed = discovery_events::resumed(&runtime, &models, &unbuilt, &state)?; + let resumed = crate::planner_migrations::_helpers::discovery::discovery_events::resumed( + &runtime, &models, &unbuilt, &state, + )?; let candidates = - discovery_requests::candidates(&runtime, &state, &names, (&declarations, &resumed))?; + crate::planner_migrations::_helpers::discovery::discovery_requests::candidates( + &runtime, + &state, + &names, + (&declarations, &resumed), + )?; for (name, _) in resumed.iter() { unbuilt.del_item(name)?; } @@ -100,13 +116,15 @@ pub(super) fn discover_model_migrations<'py>( let (variant_maps, matches, ambiguous) = match_migration_fingerprints((&models, new_names), &seeds, candidates.as_any(), &inputs)?; let mut requests = manual; - requests.extend(discovery_requests::automatic( + requests.extend(automatic( &runtime, &scope, &models, (matches.as_any(), &resumed), )?); - let warnings = discovery_requests::warnings(&models, &scope, &ambiguous)?; + let warnings = crate::planner_migrations::_helpers::discovery::discovery_requests::warnings( + &models, &scope, &ambiguous, + )?; let destination_fingerprints = migration_destination_fingerprints(&requests, &variant_maps, &inputs)?; Ok(ModelMigrationDiscovery { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs index dc9590f417..ddfd3b7cfa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs @@ -2,8 +2,8 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::planner_migrations::column_objects::config; -use crate::bindings::_helpers::planner_migrations::models::ModelMigrationDeclaration; +use crate::planner_migrations::_helpers::column::column_objects::config; +use crate::planner_migrations::models::ModelMigrationDeclaration; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -12,17 +12,17 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_move; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; -pub(super) fn identity(item: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn identity(item: &Bound<'_, PyAny>) -> PyResult { let materialized = config(item, "materialized")?; Ok(materialized.eq("table")? || materialized.eq("view")?) } -pub(super) fn history(item: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn history(item: &Bound<'_, PyAny>) -> PyResult { let materialized = config(item, "materialized")?; Ok(materialized.eq("incremental")? || materialized.eq("snapshot")?) } -pub(super) fn newest<'py>( +pub(crate) fn newest<'py>( events: &Bound<'py, PyAny>, location: &Bound<'py, PyAny>, ) -> PyResult> { @@ -31,11 +31,11 @@ pub(super) fn newest<'py>( newest_move(events, &relation) } -pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) } -pub(super) fn declared<'py>( +pub(crate) fn declared<'py>( declarations: &[ModelMigrationDeclaration<'py>], state: &Bound<'py, PyAny>, ) -> PyResult> { @@ -63,7 +63,7 @@ pub(super) fn declared<'py>( Ok(result) } -pub(super) fn renames<'py>( +pub(crate) fn renames<'py>( runtime: &Bound<'py, PyAny>, names: &Bound<'py, PyDict>, state: &Bound<'py, PyAny>, @@ -101,7 +101,7 @@ pub(super) fn renames<'py>( Ok(result) } -pub(super) fn resumed<'py>( +pub(crate) fn resumed<'py>( runtime: &Bound<'py, PyAny>, models: &[Bound<'py, PyAny>], unbuilt: &Bound<'py, PyDict>, @@ -133,7 +133,7 @@ pub(super) fn resumed<'py>( Ok(result) } -pub(super) fn key<'py>(location: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn key<'py>(location: &Bound<'py, PyAny>) -> PyResult> { let schema = native_field(location, "schema")?; let schema = if schema.is_truthy()? { schema diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_models.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_models.rs index ad16642e95..7b84223fd1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_models.rs @@ -2,9 +2,9 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::planner_identity::metadata::build_function_local_hashes; -use crate::bindings::_helpers::planner_identity::metadata::build_model_version_identity_metadata_json; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_identity::main::build_function_local_hashes::build_function_local_hashes; +use crate::planner_identity::main::build_model_version_identity_metadata_json::build_model_version_identity_metadata_json; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; @@ -21,7 +21,7 @@ use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::BTreeMap; -pub(super) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult>> { let mut models: BTreeMap> = BTreeMap::new(); let mut dependencies: BTreeMap> = BTreeMap::new(); for item in compiled_models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { @@ -43,7 +43,7 @@ pub(super) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult], scope: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>, @@ -65,7 +65,7 @@ pub(super) fn triggered( Ok(false) } -pub(super) fn metadata<'py>( +pub(crate) fn metadata<'py>( runtime: &Bound<'py, PyAny>, models: &[Bound<'py, PyAny>], ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs index 9af31f02f2..5fde1c4193 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs @@ -1,13 +1,13 @@ //! Live candidate inspection and automatic request materialization. -use crate::bindings::_helpers::planner_migrations::column_objects::config; -use crate::bindings::_helpers::planner_migrations::column_objects::native; -use crate::bindings::_helpers::planner_migrations::discovery_events::{ - identity, key, newest, relation, -}; -use crate::bindings::_helpers::planner_migrations::models::{ - ModelMigrationDeclaration, ModelMigrationRequest, -}; +use crate::planner_migrations::_helpers::column::column_objects::config; +use crate::planner_migrations::_helpers::column::column_objects::native; +use crate::planner_migrations::_helpers::discovery::discovery_events::identity; +use crate::planner_migrations::_helpers::discovery::discovery_events::key; +use crate::planner_migrations::_helpers::discovery::discovery_events::newest; +use crate::planner_migrations::_helpers::discovery::discovery_events::relation; +use crate::planner_migrations::models::ModelMigrationDeclaration; +use crate::planner_migrations::models::ModelMigrationRequest; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PySet, PySetMethods, PyString, PyTuple, @@ -49,10 +49,10 @@ fn location<'py>( ("name", name), ] }; - super::planning_origin::location(runtime, &fields) + crate::planner_migrations::_helpers::planning::planning_origin::location(runtime, &fields) } -pub(super) fn candidates<'py>( +pub(crate) fn candidates<'py>( runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, names: &Bound<'py, PyDict>, @@ -109,7 +109,7 @@ pub(super) fn candidates<'py>( PyTuple::new(py, result) } -pub(super) fn automatic<'py>( +pub(crate) fn automatic<'py>( runtime: &Bound<'py, PyAny>, scope: &Bound<'py, PyAny>, models: &[Bound<'py, PyAny>], @@ -159,7 +159,7 @@ pub(super) fn automatic<'py>( Ok(requests) } -pub(super) fn warnings<'py>( +pub(crate) fn warnings<'py>( models: &[Bound<'py, PyAny>], scope: &Bound<'py, PyAny>, ambiguous: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/mod.rs new file mode 100644 index 0000000000..bdbabf9496 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod discovery; +pub(crate) mod discovery_events; +pub(crate) mod discovery_models; +pub(crate) mod discovery_requests; +pub(crate) mod state_inspection; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs new file mode 100644 index 0000000000..6429ff9e66 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs @@ -0,0 +1,111 @@ +//! Read-only warehouse evidence gathered once for all model migrations in one plan. + +use crate::planner_migrations::models::MigrationStateInspection; +use std::collections::HashSet; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyString, PyTuple}; +use sqlbuild_state::events::main::newest_fingerprint::newest_fingerprint; +use sqlbuild_state::events::models::{EventRelation, FingerprintKey, FingerprintOrder}; + +use crate::planner_phases::main::inputs::runtime_field::runtime_field; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A lowercased schema and relation name. +pub(crate) type RelationKey = (String, String); + +pub(crate) fn lowered(value: &Bound<'_, PyAny>) -> PyResult { + value.call_method0("lower")?.extract() +} + +pub(crate) fn schema_or_empty<'py>(value: Bound<'py, PyAny>) -> Bound<'py, PyAny> { + if value.is_none() { + PyString::new(value.py(), "").into_any() + } else { + value + } +} + +/// `((schema or "").lower(), name.lower())` of a relation or location. +pub(crate) fn relation_key(value: &Bound<'_, PyAny>) -> PyResult { + Ok(( + lowered(&schema_or_empty(native_field(value, "schema")?))?, + lowered(&native_field(value, "name")?)?, + )) +} + +/// The requested schemas not inspected yet, sorted, each with its lowercased key. +pub(crate) fn pending<'py>( + schemas: &Bound<'py, PyAny>, + inspected: &HashSet, +) -> PyResult, String)>> { + let mut pending: Vec<(String, Bound<'py, PyAny>, String)> = Vec::new(); + for schema in schemas.try_iter()? { + let schema: Bound<'py, PyAny> = schema?; + let key: String = lowered(&schema)?; + if !inspected.contains(&key) { + pending.push((schema.extract()?, schema, key)); + } + } + pending.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(pending + .into_iter() + .map(|(_, schema, key)| (schema, key)) + .collect()) +} + +pub(crate) fn tuple_of<'py>(py: Python<'py>, items: &[Py]) -> PyResult> { + PyTuple::new(py, items.iter().map(|item| item.bind(py))) +} + +fn fingerprint_key(fingerprint: &Bound<'_, PyAny>) -> PyResult { + let target_name: Bound<'_, PyAny> = native_field(fingerprint, "target_name")?; + let target: Option = if target_name.is_none() { + None + } else { + Some(EventRelation { + database: native_field(fingerprint, "target_database")?.extract()?, + schema: native_field(fingerprint, "target_schema")?.extract()?, + name: target_name.extract()?, + }) + }; + Ok(FingerprintKey { + order: FingerprintOrder { + timestamp: native_field(fingerprint, "ts")? + .call_method0("isoformat")? + .extract()?, + run_id: native_field(fingerprint, "run_id")?.extract()?, + }, + node_name: native_field(fingerprint, "node_name")?.extract()?, + target, + }) +} + +pub(crate) fn newest<'py>( + py: Python<'py>, + candidates: &[Py], + (target, model_name): (Option<&EventRelation>, Option<&str>), +) -> PyResult>> { + let keys: Vec = candidates + .iter() + .map(|candidate| fingerprint_key(candidate.bind(py))) + .collect::>()?; + Ok(newest_fingerprint(&keys, target, model_name) + .map(|index| candidates[index].bind(py).clone())) +} + +/// A fresh live-state inspection over the runtime's connection and planning database. +pub(crate) fn inspection<'py>( + runtime: &Bound<'py, PyAny>, + known_fingerprints: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let state = MigrationStateInspection::new( + &runtime_field(runtime, |runtime| &runtime.adapter)?, + &runtime_field(runtime, |runtime| &runtime.connection)?, + &crate::planner_migrations::_helpers::planning::planning_origin::planning_database( + runtime, + )?, + known_fingerprints, + )?; + Ok(Bound::new(runtime.py(), state)?.into_any()) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/mod.rs new file mode 100644 index 0000000000..6815150f76 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod column; +pub(crate) mod discovery; +pub(crate) mod model; +pub(crate) mod old_name; +pub(crate) mod planning; +pub(crate) mod projections; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/mod.rs new file mode 100644 index 0000000000..b6317c802b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod model_entries; +pub(crate) mod model_handover; +pub(crate) mod model_overlay; +pub(crate) mod model_planning; +pub(crate) mod model_warnings; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs index 9e7f22bb28..66f9710514 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs @@ -2,14 +2,16 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::planner_migrations::column_objects::{lower, model, replace, text}; -use crate::bindings::_helpers::planner_migrations::compatibility::check_migration_compatibility; -use crate::bindings::_helpers::planner_migrations::model_handover::{ - applies, empty_version, handover, -}; -use crate::bindings::_helpers::planner_migrations::models::{ - CompatibilityCheck, MigrationDecisionRequest, -}; +use crate::planner_migrations::_helpers::column::column_objects::lower; +use crate::planner_migrations::_helpers::column::column_objects::model; +use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_migrations::_helpers::column::column_objects::text; +use crate::planner_migrations::_helpers::model::model_handover::applies; +use crate::planner_migrations::_helpers::model::model_handover::empty_version; +use crate::planner_migrations::_helpers::model::model_handover::handover; +use crate::planner_migrations::_helpers::planning::compatibility::check_migration_compatibility; +use crate::planner_migrations::models::CompatibilityCheck; +use crate::planner_migrations::models::MigrationDecisionRequest; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyBool; use sqlbuild_analysis::planner_migrations::main::model_decision::model_decision; @@ -24,7 +26,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_move; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; -pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) } @@ -40,7 +42,7 @@ fn key(location: &Bound<'_, PyAny>) -> PyResult<(String, String)> { )) } -pub(super) fn decision<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult> { +pub(crate) fn decision<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult> { replace( entry, &[( @@ -63,13 +65,13 @@ fn result<'py>( } /// The decided plan entry, the inherited fingerprint, and whether the history handover applies. -pub(super) struct MigrationDecision<'py> { +pub(crate) struct MigrationDecision<'py> { pub(super) entry: Bound<'py, PyAny>, pub(super) inherited: Bound<'py, PyAny>, pub(super) applies: bool, } -pub(super) fn decide<'py>( +pub(crate) fn decide<'py>( decision_request: &MigrationDecisionRequest<'_, 'py>, ) -> PyResult> { let runtime = decision_request.runtime.clone(); @@ -263,7 +265,7 @@ pub(super) fn decide<'py>( &native_field(&entry, "origin_version_hash")?, )?; result( - super::planning_execution::migration_with_execution(&runtime, &item, &entry)?, + crate::planner_migrations::_helpers::planning::planning_execution::migration_with_execution(&runtime, &item, &entry)?, inherited, true, ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs index cd6e60dc9c..cc87ead457 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs @@ -1,7 +1,8 @@ //! Adapt immutable migration fingerprints without changing their persisted representation. -use crate::bindings::_helpers::planner_migrations::column_objects::{replace, text}; -use crate::bindings::_helpers::planner_migrations::constants::MIGRATION_MODEL_NAME_METADATA_KEY; +use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_migrations::_helpers::column::column_objects::text; +use crate::planner_migrations::constants::MIGRATION_MODEL_NAME_METADATA_KEY; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::boundary::main::imported::imported; @@ -12,7 +13,7 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::q use sqlbuild_python_project::fingerprints::main::compute_query_hash::compute_query_hash; use sqlbuild_python_project::planner_phases::main::object::object; -pub(super) fn applies( +pub(crate) fn applies( model: &Bound<'_, PyAny>, event: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>, @@ -24,7 +25,7 @@ pub(super) fn applies( < presumed_utc(&native_field(event, "created_at")?)?) } -pub(super) fn handover<'py>( +pub(crate) fn handover<'py>( model: &Bound<'py, PyAny>, fingerprint: &Bound<'py, PyAny>, version: &Bound<'py, PyAny>, @@ -77,7 +78,7 @@ pub(super) fn handover<'py>( ) } -pub(super) fn equivalent<'py>( +pub(crate) fn equivalent<'py>( model: &Bound<'py, PyAny>, handover: Bound<'py, PyAny>, destinations: &Bound<'py, PyAny>, @@ -85,7 +86,11 @@ pub(super) fn equivalent<'py>( ) -> PyResult> { let py = model.py(); if handover.is_none() - || !destinations.contains(super::matching::stored_migration_fingerprint(&handover)?)? + || !destinations.contains( + crate::planner_migrations::_helpers::planning::matching::stored_migration_fingerprint( + &handover, + )?, + )? { return Ok(handover); } @@ -108,7 +113,7 @@ pub(super) fn equivalent<'py>( ) } -pub(super) fn empty_version<'py>( +pub(crate) fn empty_version<'py>( model: &Bound<'py, PyAny>, fingerprint: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs index 41931dd473..01a95ea529 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs @@ -1,15 +1,16 @@ //! Project inherited fingerprints and live relation facts before normal planning. -use crate::bindings::_helpers::planner_migrations::column_objects::native; -use crate::bindings::_helpers::planner_migrations::column_objects::{config, replace}; -use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_migrations::_helpers::column::column_objects::config; +use crate::planner_migrations::_helpers::column::column_objects::native; +use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_migrations::models::MigrationPhase; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -pub(super) fn overlay<'py>( +pub(crate) fn overlay<'py>( phase: &MigrationPhase<'py>, state: &Bound<'py, PyAny>, entries: &[Bound<'py, PyAny>], @@ -80,7 +81,7 @@ pub(super) fn overlay<'py>( let cursors = native_field(&snapshot, "cursor_snapshots")? .cast_into::()? .copy()?; - let gathered = super::planning_boundary::redirected_cursor_snapshots( + let gathered = crate::planner_migrations::_helpers::planning::planning_boundary::redirected_cursor_snapshots( phase, (relations.clone().into_any(), redirected.into_any()), None, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs similarity index 81% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs index 4f9a734749..ef56d35e05 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs @@ -1,16 +1,19 @@ //! Model migration orchestration over Python-owned live inspection. -use crate::bindings::_helpers::planner_changes::renames::plan_reference_renames; -use crate::bindings::_helpers::planner_migrations::column_objects::{config, replace}; -use crate::bindings::_helpers::planner_migrations::discovery::discover_model_migrations; -use crate::bindings::_helpers::planner_migrations::models::{ - DiscoveryRequest, MigrationDecisionRequest, MigrationPhase, ModelMigrationDeclaration, - ModelMigrationDiscovery, ModelMigrationPlanning, ModelMigrationRequest, -}; -use crate::bindings::_helpers::planner_migrations::planning_origin::{ - require_same_database, resolve_origin, -}; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_changes::main::plan_reference_renames::plan_reference_renames; +use crate::planner_migrations::_helpers::column::column_objects::config; +use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_migrations::_helpers::discovery::discovery::discover_model_migrations; +use crate::planner_migrations::_helpers::planning::planning_origin::require_same_database; +use crate::planner_migrations::_helpers::planning::planning_origin::resolve_origin; +use crate::planner_migrations::models::DiscoveryRequest; +use crate::planner_migrations::models::MigrationDecisionRequest; +use crate::planner_migrations::models::MigrationPhase; +use crate::planner_migrations::models::ModelMigrationDeclaration; +use crate::planner_migrations::models::ModelMigrationDiscovery; +use crate::planner_migrations::models::ModelMigrationPlanning; +use crate::planner_migrations::models::ModelMigrationRequest; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyString, PyTuple}; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; @@ -84,8 +87,9 @@ pub(crate) fn plan_model_migrations<'py>( ) -> PyResult> { let runtime = phase.runtime.clone(); let snapshot = phase.snapshot.clone(); - let schemas = super::planning_origin::project_schemas(&runtime)?; - let state = super::state_inspection::inspection( + let schemas = + crate::planner_migrations::_helpers::planning::planning_origin::project_schemas(&runtime)?; + let state = crate::planner_migrations::_helpers::discovery::state_inspection::inspection( &runtime, Some(&native_field(&snapshot, "fingerprints")?), )?; @@ -182,19 +186,21 @@ fn finish<'py>( } else { native_field(&project_config, "snapshots")? }; - let policy = super::planning_boundary::missing_origin_policy(&runtime)?; + let policy = crate::planner_migrations::_helpers::planning::planning_boundary::missing_origin_policy(&runtime)?; for (request, origin, origin_model) in resolved { let item = request.model.clone(); let name = name_object(&item)?; - let decided = super::model_entries::decide(&MigrationDecisionRequest { - runtime: runtime.clone(), - request, - origin, - origin_model, - snapshot: snapshot.clone(), - state: state.clone(), - snapshots_config: snapshots_config.clone(), - })?; + let decided = crate::planner_migrations::_helpers::model::model_entries::decide( + &MigrationDecisionRequest { + runtime: runtime.clone(), + request, + origin, + origin_model, + snapshot: snapshot.clone(), + state: state.clone(), + snapshots_config: snapshots_config.clone(), + }, + )?; entries.push(replace( &decided.entry, &[("missing_origin_policy", policy.clone())], @@ -207,7 +213,7 @@ fn finish<'py>( )?; handovers.set_item( name, - super::model_handover::equivalent( + crate::planner_migrations::_helpers::model::model_handover::equivalent( &item, decided.inherited, &destinations, @@ -232,14 +238,18 @@ fn finish<'py>( planned.set_item(&name, origin_model)?; } } - if let Some(warning) = super::model_warnings::entry_warning(entry)? { + if let Some(warning) = + crate::planner_migrations::_helpers::model::model_warnings::entry_warning(entry)? + { warnings.push(warning); } } let overlaid = if entries.is_empty() { snapshot.clone() } else { - super::model_overlay::overlay(phase, state, &entries, &handovers)? + crate::planner_migrations::_helpers::model::model_overlay::overlay( + phase, state, &entries, &handovers, + )? }; let reference_renames = object( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs index e2836b5359..74223c718a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs @@ -1,16 +1,15 @@ //! Adapt diagnostic display facts while preserving policy and code ordering. -use crate::bindings::_helpers::planner_migrations::column_warnings::{ - missing_origin_outcome, warning, -}; -use crate::bindings::_helpers::planner_migrations::constants::HIDDEN_ORIGIN_REMEDY; +use crate::planner_migrations::_helpers::column::column_warnings::missing_origin_outcome; +use crate::planner_migrations::_helpers::column::column_warnings::warning; +use crate::planner_migrations::constants::HIDDEN_ORIGIN_REMEDY; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyString; use sqlbuild_analysis::planner_migrations::main::model_warning::model_warning; use sqlbuild_analysis::planner_migrations::models::ModelWarning; use sqlbuild_python_base::records::main::fields::field::field as native_field; -pub(super) fn entry_warning<'py>(entry: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn entry_warning<'py>(entry: &Bound<'py, PyAny>) -> PyResult>> { let py = entry.py(); let name: String = native_field(entry, "model_name")?.extract()?; let origin = display(&native_field(entry, "origin")?)?; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/mod.rs new file mode 100644 index 0000000000..87e886af43 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod old_name_entries; +pub(crate) mod old_name_objects; +pub(crate) mod old_name_planning; +pub(crate) mod old_name_references; +pub(crate) mod old_names; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs index eb374c54d2..d220965f4a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs @@ -1,14 +1,18 @@ //! Assemble new and resumed old-name view steps from recorded facts. -use crate::bindings::_helpers::planner_migrations::column_objects::{model, text}; -use crate::bindings::_helpers::planner_migrations::constants::{PENDING_ARCHIVE, PENDING_VIEW}; -use crate::bindings::_helpers::planner_migrations::models::OldNamePlanner; -use crate::bindings::_helpers::planner_migrations::old_name_objects::{ - expires, location, relation, retention, stores_history, -}; -use crate::bindings::_helpers::planner_migrations::old_names::declared_column_aliases; -use crate::bindings::_helpers::planner_migrations::old_names::old_name_view_skip_reason; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_migrations::_helpers::column::column_objects::model; +use crate::planner_migrations::_helpers::column::column_objects::text; +use crate::planner_migrations::_helpers::old_name::old_name_objects::expires; +use crate::planner_migrations::_helpers::old_name::old_name_objects::location; +use crate::planner_migrations::_helpers::old_name::old_name_objects::relation; +use crate::planner_migrations::_helpers::old_name::old_name_objects::retention; +use crate::planner_migrations::_helpers::old_name::old_name_objects::stores_history; +use crate::planner_migrations::_helpers::old_name::old_names::declared_column_aliases; +use crate::planner_migrations::_helpers::old_name::old_names::old_name_view_skip_reason; +use crate::planner_migrations::constants::PENDING_ARCHIVE; +use crate::planner_migrations::constants::PENDING_VIEW; +use crate::planner_migrations::models::OldNamePlanner; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyBool, PyList, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs index 4f7811e34b..5eb67253db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs @@ -2,7 +2,8 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::planner_migrations::column_objects::{config, model}; +use crate::planner_migrations::_helpers::column::column_objects::config; +use crate::planner_migrations::_helpers::column::column_objects::model; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -13,11 +14,11 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::state_rows::main::migrations::old_name_retention::old_name_retention; use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; -pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) } -pub(super) fn location<'py>( +pub(crate) fn location<'py>( runtime: &Bound<'py, PyAny>, relation: &Bound<'py, PyAny>, ) -> PyResult> { @@ -34,12 +35,12 @@ pub(super) fn location<'py>( new_record_with(runtime.py(), RecordId::CompiledRelationLocation, &fields) } -pub(super) fn stores_history(item: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn stores_history(item: &Bound<'_, PyAny>) -> PyResult { let materialization = config(item, "materialized")?; Ok(materialization.eq("incremental")? || materialization.eq("snapshot")?) } -pub(super) fn retention<'py>( +pub(crate) fn retention<'py>( item: &Bound<'py, PyAny>, project: &Bound<'py, PyAny>, ) -> PyResult> { @@ -52,7 +53,7 @@ pub(super) fn retention<'py>( ) } -pub(super) fn expires<'py>( +pub(crate) fn expires<'py>( retention: &Bound<'py, PyAny>, now: &Bound<'py, PyAny>, ) -> PyResult> { @@ -65,7 +66,7 @@ pub(super) fn expires<'py>( } } -pub(super) fn recorded_view<'py>( +pub(crate) fn recorded_view<'py>( runtime: &Bound<'py, PyAny>, history: &Bound<'py, PyAny>, models: &[Bound<'py, PyAny>], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs index 4dbe2e9c79..871c2e9c66 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs @@ -2,11 +2,12 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; -use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; -use crate::bindings::_helpers::planner_migrations::old_names::old_name_claim_errors; -use crate::bindings::_helpers::planner_migrations::old_names::old_name_schemas_to_read; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_migrations::_helpers::old_name::old_name_objects::recorded_view; +use crate::planner_migrations::_helpers::old_name::old_names::old_name_claim_errors; +use crate::planner_migrations::_helpers::old_name::old_names::old_name_schemas_to_read; +use crate::planner_migrations::models::MigrationPhase; +use crate::planner_migrations::models::OldNameViewPlanning; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PySet, PySetMethods, PyTuple}; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; @@ -62,7 +63,10 @@ fn schemas<'py>( } } if include_project { - let project = super::planning_origin::project_schemas(&runtime)?; + let project = + crate::planner_migrations::_helpers::planning::planning_origin::project_schemas( + &runtime, + )?; for schema in project.try_iter()? { needed.add(schema?)?; } @@ -98,7 +102,9 @@ pub(crate) fn plan_old_name_views<'py>( selected.push(item.clone()); } } - let state = super::state_inspection::inspection(&runtime, None)?; + let state = crate::planner_migrations::_helpers::discovery::state_inspection::inspection( + &runtime, None, + )?; let needed = schemas((phase, moves.0), &selected)?; if needed.is_truthy()? { call_with( @@ -123,7 +129,7 @@ pub(crate) fn plan_old_name_views<'py>( histories.push(history); } let views = PyTuple::new(py, views)?.into_any(); - let planner = super::models::OldNamePlanner::new( + let planner = crate::planner_migrations::models::OldNamePlanner::new( (&phase.runtime, moves.1), models, histories, @@ -168,9 +174,11 @@ pub(crate) fn plan_old_name_views<'py>( )?, )?; let mut warnings: Vec> = claims.try_iter()?.collect::>()?; - warnings.extend(super::old_name_references::errors( - &runtime, &views, &entries, - )?); + warnings.extend( + crate::planner_migrations::_helpers::old_name::old_name_references::errors( + &runtime, &views, &entries, + )?, + ); Ok(OldNameViewPlanning { entries, views, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs index 4a6a085759..8b0e57d3f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs @@ -2,7 +2,8 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::planner_migrations::column_objects::{native, text}; +use crate::planner_migrations::_helpers::column::column_objects::native; +use crate::planner_migrations::_helpers::column::column_objects::text; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -49,7 +50,7 @@ fn old_relation<'py>( ) } -pub(super) fn errors<'py>( +pub(crate) fn errors<'py>( runtime: &Bound<'py, PyAny>, views: &Bound<'py, PyAny>, entries: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs index 1f5087138f..b870bbb145 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs @@ -2,7 +2,7 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::planner_migrations::old_name_objects::relation; +use crate::planner_migrations::_helpers::old_name::old_name_objects::relation; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PySet, PyTuple}; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/compatibility.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/compatibility.rs index fd9d453326..afcaf2a97d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/compatibility.rs @@ -1,9 +1,8 @@ //! Convert existing planner contracts for origin compatibility checks. -use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; -use crate::bindings::_helpers::planner_migrations::models::{ - CompatibilityCheck, MigrationCompatibilityResult, -}; +use crate::planner_changes::main::detect_model_changes::detect_model_changes; +use crate::planner_migrations::models::CompatibilityCheck; +use crate::planner_migrations::models::MigrationCompatibilityResult; use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -74,7 +73,7 @@ fn blocking<'py>( ) } /// Whether an origin relation's columns can carry the destination model's history. -pub(super) fn check_migration_compatibility<'py>( +pub(crate) fn check_migration_compatibility<'py>( check: &CompatibilityCheck<'py>, ) -> PyResult> { let py = check.model.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs index 821272dfc1..13d8a64ed0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs @@ -1,8 +1,7 @@ //! Fingerprint cache callbacks around native one-to-one migration matching. -use crate::bindings::_helpers::planner_migrations::models::{ - FingerprintInputs, ModelMigrationRequest, -}; +use crate::planner_migrations::models::FingerprintInputs; +use crate::planner_migrations::models::ModelMigrationRequest; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -46,7 +45,7 @@ fn variants<'py>( } /// The fingerprint variants tried, the matched candidate per new model, and ambiguous names. -pub(super) fn match_migration_fingerprints<'py>( +pub(crate) fn match_migration_fingerprints<'py>( (models, new_names): (&[Bound<'py, PyAny>], BTreeSet), seed_variants: &Bound<'py, PyAny>, candidates: &Bound<'py, PyAny>, @@ -86,7 +85,7 @@ pub(super) fn match_migration_fingerprints<'py>( } /// The fingerprint variants of each requested destination model with metadata. -pub(super) fn migration_destination_fingerprints<'py>( +pub(crate) fn migration_destination_fingerprints<'py>( requests: &[ModelMigrationRequest<'py>], variant_maps: &Bound<'py, PyAny>, inputs: &FingerprintInputs<'py>, @@ -108,7 +107,7 @@ pub(super) fn migration_destination_fingerprints<'py>( } /// The migration fingerprint stored in a model fingerprint's metadata. -pub(super) fn stored_migration_fingerprint( +pub(crate) fn stored_migration_fingerprint( fingerprint: &Bound<'_, PyAny>, ) -> PyResult> { let metadata: String = native_field(fingerprint, "metadata_json")?.extract()?; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/mod.rs new file mode 100644 index 0000000000..e8f2dc0ad7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod compatibility; +pub(crate) mod matching; +pub(crate) mod planning_boundary; +pub(crate) mod planning_execution; +pub(crate) mod planning_origin; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs similarity index 83% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs index 2257feefc1..9436b99e4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs @@ -1,18 +1,18 @@ //! Assemble migration policy and redirected live cursor inspection requests. -use crate::bindings::_helpers::planner_migrations::column_objects::native; -use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; -use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; -use crate::bindings::_helpers::planner_phases::retention::effective_target_config; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_warehouse::snapshot::gather_redirected_cursor_snapshots; +use crate::planner_migrations::_helpers::column::column_objects::native; +use crate::planner_migrations::models::MigrationPhase; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; +use crate::planner_phases::main::rebuilds::effective_target_config::effective_target_config; +use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; +use crate::planner_warehouse::main::inspection::gather_redirected_cursor_snapshots::gather_redirected_cursor_snapshots; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; /// The target's policy for migration origins that no longer exist. -pub(super) fn missing_origin_policy<'py>( +pub(crate) fn missing_origin_policy<'py>( runtime: &Bound<'py, PyAny>, ) -> PyResult> { native_field( @@ -22,7 +22,7 @@ pub(super) fn missing_origin_policy<'py>( } /// Live cursor snapshots of models whose history now sits in a redirected relation. -pub(super) fn redirected_cursor_snapshots<'py>( +pub(crate) fn redirected_cursor_snapshots<'py>( phase: &MigrationPhase<'py>, (existing_relations, target_relations): (Bound<'py, PyAny>, Bound<'py, PyAny>), origin_cursor_columns: Option>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs index e50f135a0e..bf918e5010 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs @@ -2,8 +2,8 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::planner_migrations::column_objects::replace; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyBool; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -40,7 +40,7 @@ fn migration_promotion<'py>( } /// The migration entry with the adapter's staging, transfer and promotion choices. -pub(super) fn migration_with_execution<'py>( +pub(crate) fn migration_with_execution<'py>( runtime: &Bound<'py, PyAny>, model: &Bound<'py, PyAny>, entry: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs index 53fb78acba..c303de0661 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs @@ -1,8 +1,8 @@ //! Resolve authored migration origins at the adapter and inspection boundaries. -use crate::bindings::_helpers::planner_migrations::constants::QUALIFIED_RELATION_MAX_PARTS; -use crate::bindings::_helpers::planner_migrations::models::ModelMigrationRequest; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_migrations::constants::QUALIFIED_RELATION_MAX_PARTS; +use crate::planner_migrations::models::ModelMigrationRequest; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::PyErr; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PySet, PySetMethods}; @@ -15,7 +15,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; -pub(super) fn location<'py>( +pub(crate) fn location<'py>( runtime: &Bound<'py, PyAny>, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { @@ -68,7 +68,7 @@ fn input_error(py: Python<'_>, message: String) -> PyResult { } /// The database migration planning inspects: the first model destination's, else the target's. -pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { let project = runtime_field(runtime, |runtime| &runtime.project)?; let fallback = native_field(&project, "effective_target_database")?; for item in compiled_models(&project)?.try_iter()? { @@ -81,7 +81,7 @@ pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult(runtime: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn project_schemas<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { let schemas = PySet::empty(runtime.py())?; for item in compiled_models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let schema = native_field(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&item?)?, "schema")?; @@ -93,7 +93,7 @@ pub(super) fn project_schemas<'py>(runtime: &Bound<'py, PyAny>) -> PyResult, location: &Bound<'_, PyAny>, ) -> PyResult<()> { @@ -124,7 +124,7 @@ pub(super) fn require_same_database( } /// The origin location and origin model name of one migration request. -pub(super) fn resolve_origin<'py>( +pub(crate) fn resolve_origin<'py>( request: &ModelMigrationRequest<'py>, runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, @@ -135,7 +135,7 @@ pub(super) fn resolve_origin<'py>( } let raw = match &request.raw_origin { Some(raw) if raw.is_truthy()? => raw.clone(), - _ => super::column_objects::text(py, ""), + _ => crate::planner_migrations::_helpers::column::column_objects::text(py, ""), }; let item = request.model.clone(); let destination = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/getters.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/getters.rs similarity index 66% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/getters.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/getters.rs index 92bf934e48..6901d0c07c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/getters.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/getters.rs @@ -3,17 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::planner_migrations::state_inspection::MigrationStateInspection; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::planner_migrations::models::MigrationStateInspection; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { - class: class_of::, - read: read_migration_state_inspection, -}]; - -fn read_migration_state_inspection<'py>( +pub(crate) fn read_migration_state_inspection<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/mod.rs new file mode 100644 index 0000000000..4db25c4cea --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod fingerprint; +pub(crate) mod getters; +pub(crate) mod projections; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/projections.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/projections.rs index d5a131042e..63a466f24c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/projections.rs @@ -1,6 +1,6 @@ //! Adapt existing Python query shape models to pure migration proofs. -use crate::bindings::_helpers::planner_migrations::errors::ProofError; +use crate::planner_migrations::errors::ProofError; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/constants.rs b/crates/sqlbuild-python-planner/src/planner_migrations/constants.rs new file mode 100644 index 0000000000..06519f2cb1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/constants.rs @@ -0,0 +1,38 @@ +//! Public constants of planner migrations. + +use crate::planner_migrations::_helpers::projections::getters::read_migration_state_inspection; +use crate::planner_migrations::models::MigrationStateInspection; +use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +pub(crate) const AUTOMATIC: &str = "automatic"; + +pub(crate) const DONE: &str = "done"; + +pub(crate) const PENDING_ARCHIVE: &str = "pending_archive"; + +pub(crate) const PENDING_VIEW: &str = "pending_view"; + +pub(crate) const LIST_RELATIONS_FOR_INSPECTION: AdapterHost = + AdapterHost::ListRelationsForInspection; + +pub(crate) const GET_COLUMNS_FOR_INSPECTION: AdapterHost = AdapterHost::GetColumnsForInspection; + +pub(crate) const QUALIFIED_RELATION_MAX_PARTS: usize = 3; + +pub(crate) const MIGRATION_MODEL_NAME_METADATA_KEY: &str = "model_name"; + +pub(crate) const HIDDEN_ORIGIN_REMEDY: &str = "the connection may lack permission to see it, or it was dropped outside SQLBuild; restore access to it, or remove migrate_from to build the model fresh deliberately (missing_migration_origin does not apply)"; + +pub(crate) const MISSING_ORIGIN_ALLOW_OUTCOME: &str = "nothing is migrated and the build continues as if migrate_from were absent; remove migrate_from once no target still needs it"; + +pub(crate) const MISSING_ORIGIN_CONFIRMATION_OUTCOME: &str = "building without it needs confirmation (missing_migration_origin = require_confirmation); pass --allow-missing-migration-origin, or remove migrate_from if it is no longer needed"; + +pub(crate) const MISSING_ORIGIN_DENY_OUTCOME: &str = "the build stops (missing_migration_origin = deny); if the migration already happened elsewhere or is no longer needed, remove migrate_from"; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[GetterEntry { + class: class_of::, + read: read_migration_state_inspection, +}]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/errors.rs b/crates/sqlbuild-python-planner/src/planner_migrations/errors.rs similarity index 69% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/errors.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/errors.rs index 2f26486686..53c8c47d73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/errors.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/errors.rs @@ -1,4 +1,4 @@ -//! Preserve input lookup exceptions across the pure proof callback. +//! Errors of migration planning: input lookup exceptions preserved across the pure proof callback. use pyo3::{PyErr, exceptions::PyValueError}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/fingerprint.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/fingerprint.rs new file mode 100644 index 0000000000..976920e395 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/fingerprint.rs @@ -0,0 +1,16 @@ +//! The migration fingerprint of SQL and metadata through rename identities, `generic` by default. + +use crate::planner_migrations::_helpers::projections::fingerprint; +use pyo3::prelude::PyResult; +use std::collections::BTreeMap; + +/// The migration fingerprint of SQL and metadata through rename identities, `generic` by default. +#[inline] +pub fn fingerprint( + sql: &str, + metadata: &str, + identities: &BTreeMap, + dialect: Option<&str>, +) -> PyResult> { + fingerprint::fingerprint(sql, metadata, identities, dialect) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/migration_fingerprint_ref_names.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/migration_fingerprint_ref_names.rs new file mode 100644 index 0000000000..5d53694370 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/migration_fingerprint_ref_names.rs @@ -0,0 +1,17 @@ +//! Every name whose rename identity can change the migration fingerprint. + +use crate::planner_migrations::_helpers::projections::fingerprint; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyFrozenSet; + +/// Every name whose rename identity can change the migration fingerprint. +#[inline] +pub fn migration_fingerprint_ref_names<'py>( + py: Python<'py>, + query_sql: &str, + metadata_json: &str, +) -> PyResult> { + fingerprint::migration_fingerprint_ref_names(py, query_sql, metadata_json) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/mod.rs new file mode 100644 index 0000000000..972c3e0206 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/mod.rs @@ -0,0 +1,10 @@ +pub mod fingerprint; +pub mod migration_fingerprint_ref_names; +pub mod plan_column_migrations; +pub mod plan_model_migrations; +pub mod plan_old_name_views; +pub mod register_column_planning; +pub mod register_columns; +pub mod register_fingerprint; +pub mod register_projections; +pub mod with_migration_fingerprints; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_column_migrations.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_column_migrations.rs new file mode 100644 index 0000000000..6f49497215 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_column_migrations.rs @@ -0,0 +1,16 @@ +//! Decide column renames for the selected incremental and snapshot models. + +use crate::planner_migrations::_helpers::column::column_planning; +use crate::planner_migrations::models::ColumnMigrationInputs; +use crate::planner_migrations::models::ColumnMigrationPlanning; +use crate::planner_migrations::models::MigrationPhase; +use pyo3::prelude::PyResult; + +/// Decide column renames for the selected incremental and snapshot models. +#[inline] +pub fn plan_column_migrations<'py>( + phase: &MigrationPhase<'py>, + columns: &ColumnMigrationInputs<'py>, +) -> PyResult> { + column_planning::plan_column_migrations(phase, columns) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_model_migrations.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_model_migrations.rs new file mode 100644 index 0000000000..21dfde2f07 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_model_migrations.rs @@ -0,0 +1,17 @@ +//! Model migrations of the selected scope: discovered and declared moves, decided against live state. + +use crate::planner_migrations::_helpers::model::model_planning; +use crate::planner_migrations::models::MigrationPhase; +use crate::planner_migrations::models::ModelMigrationPlanning; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Model migrations of the selected scope: discovered and declared moves, decided against live state. +#[inline] +pub fn plan_model_migrations<'py>( + phase: &MigrationPhase<'py>, + fingerprints: &Bound<'py, PyAny>, +) -> PyResult> { + model_planning::plan_model_migrations(phase, fingerprints) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_old_name_views.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_old_name_views.rs new file mode 100644 index 0000000000..a5ac9ef575 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_old_name_views.rs @@ -0,0 +1,17 @@ +//! Old-name compatibility views to create, resume or retire for the planned model and column migrations. + +use crate::planner_migrations::_helpers::old_name::old_name_planning; +use crate::planner_migrations::models::MigrationPhase; +use crate::planner_migrations::models::OldNameViewPlanning; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Old-name compatibility views to create, resume or retire for the planned model and column migrations. +#[inline] +pub fn plan_old_name_views<'py>( + phase: &MigrationPhase<'py>, + moves: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + old_name_planning::plan_old_name_views(phase, moves) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_column_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_column_planning.rs new file mode 100644 index 0000000000..cff40e933b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_column_planning.rs @@ -0,0 +1,12 @@ +//! Register the column planning functions and classes. + +use crate::planner_migrations::_helpers::column::column_planning; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the column planning functions and classes. +#[inline] +pub fn register_column_planning(module: &Bound<'_, PyModule>) -> PyResult<()> { + column_planning::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_columns.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_columns.rs new file mode 100644 index 0000000000..f1bcf4f3f1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_columns.rs @@ -0,0 +1,12 @@ +//! Register the columns functions and classes. + +use crate::planner_migrations::_helpers::column::columns; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the columns functions and classes. +#[inline] +pub fn register_columns(module: &Bound<'_, PyModule>) -> PyResult<()> { + columns::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_fingerprint.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_fingerprint.rs new file mode 100644 index 0000000000..3bd9e3d38b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_fingerprint.rs @@ -0,0 +1,12 @@ +//! Register the fingerprint functions and classes. + +use crate::planner_migrations::_helpers::projections::fingerprint; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the fingerprint functions and classes. +#[inline] +pub fn register_fingerprint(module: &Bound<'_, PyModule>) -> PyResult<()> { + fingerprint::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_projections.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_projections.rs new file mode 100644 index 0000000000..f6a8f4a0ca --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_projections.rs @@ -0,0 +1,12 @@ +//! Register the projections functions and classes. + +use crate::planner_migrations::_helpers::projections::projections; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the projections functions and classes. +#[inline] +pub fn register_projections(module: &Bound<'_, PyModule>) -> PyResult<()> { + projections::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs new file mode 100644 index 0000000000..22c55b847b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs @@ -0,0 +1,19 @@ +//! Attach persisted migration identities through the existing cache boundary. + +use crate::planner_migrations::_helpers::projections::fingerprint; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; +use sqlbuild_python_project::native_host::models::ModelLookup; + +/// Attach persisted migration identities through the existing cache boundary. +#[inline] +pub fn with_migration_fingerprints<'py>( + entries: &Bound<'py, PyAny>, + models: &ModelLookup, + cache: &Bound<'py, PyAny>, + dialect: &Bound<'py, PyAny>, +) -> PyResult> { + fingerprint::with_migration_fingerprints(entries, models, cache, dialect) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/mod.rs new file mode 100644 index 0000000000..4dfc016833 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/mod.rs @@ -0,0 +1,7 @@ +//! Column and model migration planning: candidates, entries, overlays, handover and old names. + +pub(crate) mod _helpers; +pub mod constants; +pub(crate) mod errors; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs similarity index 67% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/models.rs index 820ea2335c..9a11795015 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs @@ -1,143 +1,226 @@ -//! Read-only warehouse evidence gathered once for all model migrations in one plan. +//! Public records of planner migrations. -use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; -use std::collections::{HashMap, HashSet}; - -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyRef, PyRefMut, PyResult, Python, pyclass, pymethods, -}; -use pyo3::types::{PyBool, PyDict, PyString, PyTuple, PyTupleMethods}; +use crate::planner_migrations::_helpers::discovery::state_inspection::RelationKey; +use crate::planner_migrations::_helpers::discovery::state_inspection::lowered; +use crate::planner_migrations::_helpers::discovery::state_inspection::newest; +use crate::planner_migrations::_helpers::discovery::state_inspection::pending; +use crate::planner_migrations::_helpers::discovery::state_inspection::relation_key; +use crate::planner_migrations::_helpers::discovery::state_inspection::schema_or_empty; +use crate::planner_migrations::_helpers::discovery::state_inspection::tuple_of; +use crate::planner_migrations::constants::GET_COLUMNS_FOR_INSPECTION; +use crate::planner_migrations::constants::LIST_RELATIONS_FOR_INSPECTION; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyRef; +use pyo3::prelude::PyRefMut; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::prelude::pyclass; +use pyo3::prelude::pymethods; +use pyo3::types::PyBool; +use pyo3::types::PyDict; +use pyo3::types::PyString; +use pyo3::types::PyTuple; +use pyo3::types::PyTupleMethods; +use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_MODEL; use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_core::constants::MIGRATION_TABLE_NAME; use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; -use sqlbuild_state::events::main::newest_fingerprint::newest_fingerprint; -use sqlbuild_state::events::models::{EventRelation, FingerprintKey, FingerprintOrder}; - -use crate::bindings::_helpers::planner_migrations::constants::{ - GET_COLUMNS_FOR_INSPECTION, LIST_RELATIONS_FOR_INSPECTION, -}; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_MODEL; -use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; -use sqlbuild_python_front::state_rows::constants::{COLUMN_KIND, MODEL_KIND, OLD_NAME_KIND}; +use sqlbuild_python_front::state_rows::constants::COLUMN_KIND; +use sqlbuild_python_front::state_rows::constants::MODEL_KIND; +use sqlbuild_python_front::state_rows::constants::OLD_NAME_KIND; use sqlbuild_python_front::state_rows::main::migrations::layout::layout; use sqlbuild_python_front::state_rows::main::migrations::old_name_views::old_name_views; use sqlbuild_python_front::state_rows::main::migrations::read_events::read_events; use sqlbuild_python_front::state_rows::main::migrations::stored_old_name_columns::stored_old_name_columns; use sqlbuild_python_front::state_rows::models::StateAccess; +use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; +use sqlbuild_state::events::models::EventRelation; +use std::collections::HashMap; +use std::collections::HashSet; -/// A lowercased schema and relation name. -type RelationKey = (String, String); +#[derive(Debug)] +pub(crate) struct OldNamePlanner<'py> { + pub runtime: Bound<'py, PyAny>, + pub models: Vec>, + pub histories: Vec>, + pub views: Bound<'py, PyAny>, + pub columns: Bound<'py, PyAny>, + pub retention: Bound<'py, PyAny>, + pub now: Bound<'py, PyAny>, +} -/// Relations, fingerprints and migration events for one planning database. -#[pyclass(module = "sqlbuild._native", name = "MigrationStateInspection")] -pub(crate) struct MigrationStateInspection { - adapter: Py, - connection: Py, - database: Py, - known_fingerprints: Py, - read_schemas: HashSet, - /// Model fingerprints per lowercased schema, in first-read order. - fingerprints: Vec<(String, Vec>)>, - events: Vec>, - event_schemas: HashSet, - relations: HashMap>, - columns: HashMap>, - column_event_schemas: HashSet, - column_events: Vec>, - old_name_schemas: HashSet, - old_name_moves: Vec>, - old_name_facts: Vec>, +#[derive(Debug)] +pub(crate) struct ColumnCandidate<'py> { + pub model: Bound<'py, PyAny>, + pub warehouse: Bound<'py, PyDict>, + pub declared: Bound<'py, PyDict>, + pub fingerprint: Bound<'py, PyAny>, + pub previous: Bound<'py, PyAny>, + pub current: Bound<'py, PyAny>, } -fn lowered(value: &Bound<'_, PyAny>) -> PyResult { - value.call_method0("lower")?.extract() +#[derive(Debug)] +pub(crate) struct CheckedColumns<'py> { + pub entries: Vec>, + pub warnings: Vec>, } -fn schema_or_empty<'py>(value: Bound<'py, PyAny>) -> Bound<'py, PyAny> { - if value.is_none() { - PyString::new(value.py(), "").into_any() - } else { - value - } +/// The planning-phase inputs every migration planner reads. +#[derive(Debug)] +pub struct MigrationPhase<'py> { + pub runtime: Bound<'py, PyAny>, + pub scope: Bound<'py, PyAny>, + pub snapshot: Bound<'py, PyAny>, + pub overrides: Bound<'py, PyAny>, + pub deferral: Bound<'py, PyAny>, } -/// `((schema or "").lower(), name.lower())` of a relation or location. -fn relation_key(value: &Bound<'_, PyAny>) -> PyResult { - Ok(( - lowered(&schema_or_empty(native_field(value, "schema")?))?, - lowered(&native_field(value, "name")?)?, - )) +/// The column migration planning inputs beyond the phase. +#[derive(Debug)] +pub struct ColumnMigrationInputs<'py> { + pub full_refresh_model_names: Bound<'py, PyAny>, + pub physical_relations: Bound<'py, PyAny>, + pub source_columns: Bound<'py, PyAny>, } -/// The requested schemas not inspected yet, sorted, each with its lowercased key. -fn pending<'py>( - schemas: &Bound<'py, PyAny>, - inspected: &HashSet, -) -> PyResult, String)>> { - let mut pending: Vec<(String, Bound<'py, PyAny>, String)> = Vec::new(); - for schema in schemas.try_iter()? { - let schema: Bound<'py, PyAny> = schema?; - let key: String = lowered(&schema)?; - if !inspected.contains(&key) { - pending.push((schema.extract()?, schema, key)); - } - } - pending.sort_by(|left, right| left.0.cmp(&right.0)); - Ok(pending - .into_iter() - .map(|(_, schema, key)| (schema, key)) - .collect()) +/// The snapshot after model migrations overlay it, with the planned entries and their warnings. +#[derive(Debug)] +pub struct ModelMigrationPlanning<'py> { + pub snapshot: Bound<'py, PyAny>, + pub entries: Vec>, + pub warnings: Vec>, } -fn tuple_of<'py>(py: Python<'py>, items: &[Py]) -> PyResult> { - PyTuple::new(py, items.iter().map(|item| item.bind(py))) +/// The snapshot after column renames overlay it, with the planned entries, rename hints and warnings. +#[derive(Debug)] +pub struct ColumnMigrationPlanning<'py> { + pub snapshot: Bound<'py, PyAny>, + pub entries: Vec>, + pub hints: Vec>, + pub warnings: Vec>, } -fn fingerprint_key(fingerprint: &Bound<'_, PyAny>) -> PyResult { - let target_name: Bound<'_, PyAny> = native_field(fingerprint, "target_name")?; - let target: Option = if target_name.is_none() { - None - } else { - Some(EventRelation { - database: native_field(fingerprint, "target_database")?.extract()?, - schema: native_field(fingerprint, "target_schema")?.extract()?, - name: target_name.extract()?, - }) - }; - Ok(FingerprintKey { - order: FingerprintOrder { - timestamp: native_field(fingerprint, "ts")? - .call_method0("isoformat")? - .extract()?, - run_id: native_field(fingerprint, "run_id")?.extract()?, - }, - node_name: native_field(fingerprint, "node_name")?.extract()?, - target, - }) +/// The old-name view steps, the recorded live views and the warnings they raise. +#[derive(Debug)] +pub struct OldNameViewPlanning<'py> { + pub entries: Bound<'py, PyAny>, + pub views: Bound<'py, PyAny>, + pub warnings: Vec>, } -fn newest<'py>( - py: Python<'py>, - candidates: &[Py], - (target, model_name): (Option<&EventRelation>, Option<&str>), -) -> PyResult>> { - let keys: Vec = candidates - .iter() - .map(|candidate| fingerprint_key(candidate.bind(py))) - .collect::>()?; - Ok(newest_fingerprint(&keys, target, model_name) - .map(|index| candidates[index].bind(py).clone())) +/// One destination model whose data should come from an earlier relation. +#[derive(Debug, Clone)] +pub(crate) struct ModelMigrationRequest<'py> { + pub model: Bound<'py, PyAny>, + pub discovery: Bound<'py, PyAny>, + pub raw_origin: Option>, + pub origin_location: Option>, + pub origin_model: Bound<'py, PyAny>, + pub force: bool, + pub identity_only: bool, +} + +/// One project model's explicit migrate_from, resolved to a relation. +#[derive(Debug)] +pub(crate) struct ModelMigrationDeclaration<'py> { + pub model_name: Bound<'py, PyAny>, + pub origin_location: Bound<'py, PyAny>, + pub origin_model: Bound<'py, PyAny>, +} + +/// Manual and automatically discovered migration requests for one plan. +#[derive(Debug)] +pub(crate) struct ModelMigrationDiscovery<'py> { + pub requests: Vec>, + pub warnings: Vec>, + pub destination_fingerprints: Bound<'py, PyDict>, +} + +/// Compatibility status plus human-readable blocking findings. +#[derive(Debug)] +pub(crate) struct MigrationCompatibilityResult<'py> { + pub status: Bound<'py, PyAny>, + pub findings: Bound<'py, PyAny>, +} + +/// The fingerprint inputs shared by migration matching and destination fingerprints. +#[derive(Debug)] +pub(crate) struct FingerprintInputs<'py> { + pub metadata_jsons: Bound<'py, PyAny>, + pub dialect: Bound<'py, PyAny>, + pub fingerprints: Bound<'py, PyAny>, +} + +/// One resolved migration request decided against live state. +#[derive(Debug)] +pub(crate) struct MigrationDecisionRequest<'a, 'py> { + pub runtime: Bound<'py, PyAny>, + pub request: &'a ModelMigrationRequest<'py>, + pub origin: Bound<'py, PyAny>, + pub origin_model: Bound<'py, PyAny>, + pub snapshot: Bound<'py, PyAny>, + pub state: Bound<'py, PyAny>, + pub snapshots_config: Bound<'py, PyAny>, +} + +/// What checking an origin relation's compatibility with its destination model reads. +#[derive(Debug)] +pub(crate) struct CompatibilityCheck<'py> { + pub model: Bound<'py, PyAny>, + pub origin_relation: Bound<'py, PyAny>, + pub origin_columns: Bound<'py, PyAny>, + pub sql_analysis_enabled: Bound<'py, PyAny>, + pub column_dialect: Bound<'py, PyAny>, + pub snapshots_config: Bound<'py, PyAny>, +} + +/// What discovering automatic and manual migration requests reads. +#[derive(Debug)] +pub(crate) struct DiscoveryRequest<'py> { + pub runtime: Bound<'py, PyAny>, + pub scope: Bound<'py, PyAny>, + pub snapshot: Bound<'py, PyAny>, + pub manual_requests: Vec>, + pub declarations: Vec>, + pub state: Bound<'py, PyAny>, + pub project_schemas: Bound<'py, PyAny>, + pub fingerprints: Bound<'py, PyAny>, +} + +/// Relations, fingerprints and migration events for one planning database. +#[pyclass(module = "sqlbuild._native", name = "MigrationStateInspection")] +#[derive(Debug)] +pub struct MigrationStateInspection { + pub(crate) adapter: Py, + pub(crate) connection: Py, + pub(crate) database: Py, + pub(crate) known_fingerprints: Py, + pub(crate) read_schemas: HashSet, + /// Model fingerprints per lowercased schema, in first-read order. + pub(crate) fingerprints: Vec<(String, Vec>)>, + pub(crate) events: Vec>, + pub(crate) event_schemas: HashSet, + pub(crate) relations: HashMap>, + pub(crate) columns: HashMap>, + pub(crate) column_event_schemas: HashSet, + pub(crate) column_events: Vec>, + pub(crate) old_name_schemas: HashSet, + pub(crate) old_name_moves: Vec>, + pub(crate) old_name_facts: Vec>, } impl MigrationStateInspection { - fn access<'py>(&self, py: Python<'py>) -> PyResult> { + pub(crate) fn access<'py>(&self, py: Python<'py>) -> PyResult> { let adapter: &Bound<'py, PyAny> = self.adapter.bind(py); Ok(StateAccess { connection: self.connection.bind(py).clone(), @@ -146,7 +229,7 @@ impl MigrationStateInspection { }) } - fn list_relations<'py>( + pub(crate) fn list_relations<'py>( &self, py: Python<'py>, schemas: Vec>, @@ -166,7 +249,7 @@ impl MigrationStateInspection { .collect() } - fn model_fingerprints<'py>( + pub(crate) fn model_fingerprints<'py>( &self, py: Python<'py>, schema: &Bound<'py, PyAny>, @@ -218,7 +301,7 @@ impl MigrationStateInspection { impl MigrationStateInspection { #[new] #[pyo3(signature = (*, adapter, connection, database, known_fingerprints = None))] - fn new( + pub(crate) fn new( adapter: &Bound<'_, PyAny>, connection: &Bound<'_, PyAny>, database: &Bound<'_, PyAny>, @@ -255,25 +338,25 @@ impl MigrationStateInspection { /// The database whose namespaces this inspection reads. #[getter] - pub(crate) fn database(&self, py: Python<'_>) -> Py { + pub fn database(&self, py: Python<'_>) -> Py { self.database.clone_ref(py) } /// Every migration event read so far. #[getter] - pub(crate) fn events<'py>(&self, py: Python<'py>) -> PyResult> { + pub fn events<'py>(&self, py: Python<'py>) -> PyResult> { tuple_of(py, &self.events) } /// Every column migration event read so far. #[getter] - pub(crate) fn column_events<'py>(&self, py: Python<'py>) -> PyResult> { + pub fn column_events<'py>(&self, py: Python<'py>) -> PyResult> { tuple_of(py, &self.column_events) } /// The old-name history of every move read by `inspect_old_name_views`. #[getter] - pub(crate) fn old_name_histories<'py>(&self, py: Python<'py>) -> PyResult> { + pub fn old_name_histories<'py>(&self, py: Python<'py>) -> PyResult> { old_name_views( tuple_of(py, &self.old_name_moves)?.as_any(), tuple_of(py, &self.old_name_facts)?.as_any(), @@ -282,7 +365,10 @@ impl MigrationStateInspection { /// Read the column migration state table once for every schema not inspected yet. #[pyo3(signature = (*, schemas))] - fn inspect_column_events(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + pub(crate) fn inspect_column_events( + slf: &Bound<'_, Self>, + schemas: &Bound<'_, PyAny>, + ) -> PyResult<()> { let py: Python<'_> = slf.py(); let pending = pending(schemas, &slf.borrow().column_event_schemas)?; if pending.is_empty() { @@ -322,7 +408,10 @@ impl MigrationStateInspection { /// Read moves and old-name facts once for every schema not inspected yet. #[pyo3(signature = (*, schemas))] - fn inspect_old_name_views(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + pub(crate) fn inspect_old_name_views( + slf: &Bound<'_, Self>, + schemas: &Bound<'_, PyAny>, + ) -> PyResult<()> { let py: Python<'_> = slf.py(); let pending = pending(schemas, &slf.borrow().old_name_schemas)?; if pending.is_empty() { @@ -376,7 +465,10 @@ impl MigrationStateInspection { /// Read migration events once from schemas known to hold the migration table. #[pyo3(signature = (*, schemas))] - fn inspect_migration_events(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + pub(crate) fn inspect_migration_events( + slf: &Bound<'_, Self>, + schemas: &Bound<'_, PyAny>, + ) -> PyResult<()> { let py: Python<'_> = slf.py(); let mut sorted: Vec<(String, Bound<'_, PyAny>)> = Vec::new(); for schema in schemas.try_iter()? { @@ -403,7 +495,10 @@ impl MigrationStateInspection { /// Read state tables once for every schema not inspected yet. #[pyo3(signature = (*, schemas))] - fn inspect_schemas(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + pub(crate) fn inspect_schemas( + slf: &Bound<'_, Self>, + schemas: &Bound<'_, PyAny>, + ) -> PyResult<()> { let py: Python<'_> = slf.py(); let pending = pending(schemas, &slf.borrow().read_schemas)?; if pending.is_empty() { @@ -442,7 +537,10 @@ impl MigrationStateInspection { /// List candidate relations and their columns with one metadata read. #[pyo3(signature = (*, locations))] - fn inspect_relations(slf: &Bound<'_, Self>, locations: &Bound<'_, PyAny>) -> PyResult<()> { + pub(crate) fn inspect_relations( + slf: &Bound<'_, Self>, + locations: &Bound<'_, PyAny>, + ) -> PyResult<()> { let py: Python<'_> = slf.py(); let mut wanted: HashSet = HashSet::new(); let mut schemas: Vec = Vec::new(); @@ -513,7 +611,11 @@ impl MigrationStateInspection { } /// The listed relation for a location. - fn relation(&self, py: Python<'_>, location: &Bound<'_, PyAny>) -> PyResult> { + pub(crate) fn relation( + &self, + py: Python<'_>, + location: &Bound<'_, PyAny>, + ) -> PyResult> { Ok(match self.relations.get(&relation_key(location)?) { Some(relation) => relation.clone_ref(py), None => py.None(), @@ -521,7 +623,11 @@ impl MigrationStateInspection { } /// Listed columns for a location. - fn relation_columns(&self, py: Python<'_>, location: &Bound<'_, PyAny>) -> PyResult> { + pub(crate) fn relation_columns( + &self, + py: Python<'_>, + location: &Bound<'_, PyAny>, + ) -> PyResult> { Ok(match self.columns.get(&relation_key(location)?) { Some(columns) => columns.clone_ref(py), None => PyTuple::empty(py).into_any().unbind(), @@ -530,7 +636,7 @@ impl MigrationStateInspection { /// The latest model fingerprint that built the location or named model. #[pyo3(signature = (*, location, model_name))] - fn model_fingerprint<'py>( + pub(crate) fn model_fingerprint<'py>( &self, py: Python<'py>, location: &Bound<'py, PyAny>, @@ -551,7 +657,7 @@ impl MigrationStateInspection { /// The newest model fingerprint with this node name in any inspected schema. #[pyo3(signature = (*, model_name))] - fn named_fingerprint<'py>( + pub(crate) fn named_fingerprint<'py>( &self, py: Python<'py>, model_name: &str, @@ -565,7 +671,7 @@ impl MigrationStateInspection { /// Latest model fingerprints whose names are no longer in the project. #[pyo3(signature = (*, project_model_names))] - fn orphan_fingerprints<'py>( + pub(crate) fn orphan_fingerprints<'py>( &self, py: Python<'py>, project_model_names: &Bound<'py, PyAny>, @@ -582,17 +688,3 @@ impl MigrationStateInspection { PyTuple::new(py, orphans) } } - -/// A fresh live-state inspection over the runtime's connection and planning database. -pub(super) fn inspection<'py>( - runtime: &Bound<'py, PyAny>, - known_fingerprints: Option<&Bound<'py, PyAny>>, -) -> PyResult> { - let state = MigrationStateInspection::new( - &runtime_field(runtime, |runtime| &runtime.adapter)?, - &runtime_field(runtime, |runtime| &runtime.connection)?, - &super::planning_origin::planning_database(runtime)?, - known_fingerprints, - )?; - Ok(Bound::new(runtime.py(), state)?.into_any()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/contexts.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/contexts.rs index 106761dc8c..68a9ee4db6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/contexts.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, Py, PyAny, PyResult}; use pyo3::pymethods; use pyo3::{PyClass, pyclass::boolean_struct::True}; -use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; +use crate::planner_phases::models::PlannerIdentityContext; #[pymethods] impl PlannerIdentityContext { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs similarity index 76% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs index 8aa33875fe..3bbbc018fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs @@ -3,39 +3,13 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::planner_phases::models::{ - PlannerIdentityContext, PlannerRuntime, PlannerWarehouseState, -}; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::planner_phases::models::PlannerIdentityContext; +use crate::planner_phases::models::PlannerRuntime; +use crate::planner_phases::models::PlannerWarehouseState; use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use sqlbuild_python_project::planner_scope::models::RetainedScope; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[ - GetterEntry { - class: class_of::, - read: read_retained_scope, - }, - GetterEntry { - class: class_of::, - read: read_planner_identity_context, - }, - GetterEntry { - class: class_of::, - read: read_planner_runtime, - }, - GetterEntry { - class: class_of::, - read: read_planner_scope_resolution, - }, - GetterEntry { - class: class_of::, - read: read_planner_warehouse_state, - }, -]; - -fn read_retained_scope<'py>( +pub(crate) fn read_retained_scope<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -45,7 +19,7 @@ fn read_retained_scope<'py>( Some(cell.get().data.field(value.py(), name)) } -fn read_planner_identity_context<'py>( +pub(crate) fn read_planner_identity_context<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -63,7 +37,7 @@ fn read_planner_identity_context<'py>( }) } -fn read_planner_runtime<'py>( +pub(crate) fn read_planner_runtime<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -86,7 +60,7 @@ fn read_planner_runtime<'py>( }) } -fn read_planner_scope_resolution<'py>( +pub(crate) fn read_planner_scope_resolution<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -105,7 +79,7 @@ fn read_planner_scope_resolution<'py>( }) } -fn read_planner_warehouse_state<'py>( +pub(crate) fn read_planner_warehouse_state<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/mod.rs new file mode 100644 index 0000000000..c7081fc170 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod contexts; +pub(crate) mod getters; +pub(crate) mod runtime; +pub(crate) mod warehouse_state; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs index 2439da6be4..cf1926333b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs @@ -1,11 +1,12 @@ //! The planner runtime and warehouse state: built from keyword arguments and read by planner phases. -use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; +use crate::planner_warehouse::models::MigrationFingerprintCache; use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; use pyo3::types::{PyDict, PyTuple}; -use crate::bindings::_helpers::planner_phases::models::{PlannerRuntime, PlannerWarehouseState}; +use crate::planner_phases::models::PlannerRuntime; +use crate::planner_phases::models::PlannerWarehouseState; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::build_command::main::utc_now::utc_now; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs index 1150401e9e..07ebe967f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs @@ -1,19 +1,19 @@ //! The warehouse inspection phase: snapshot, migrations and inspection relations in one pass. -use crate::bindings::_helpers::planner_entries::models::RelationsRequest; -use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; -use crate::bindings::_helpers::planner_migrations::column_planning::plan_column_migrations; -use crate::bindings::_helpers::planner_migrations::model_planning::plan_model_migrations; -use crate::bindings::_helpers::planner_migrations::models::{ - ColumnMigrationInputs, MigrationPhase, -}; -use crate::bindings::_helpers::planner_migrations::old_name_planning::plan_old_name_views as plan_old_name_migration_views; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; -use crate::bindings::_helpers::planner_phases::models::PlannerWarehouseState; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; -use crate::bindings::_helpers::planner_warehouse::snapshot::{SnapshotRequest, warehouse_snapshot}; +use crate::planner_entries::main::entries::planner_relations_context::planner_relations_context; +use crate::planner_entries::models::RelationsRequest; +use crate::planner_migrations::main::plan_column_migrations::plan_column_migrations; +use crate::planner_migrations::main::plan_model_migrations::plan_model_migrations; +use crate::planner_migrations::main::plan_old_name_views::plan_old_name_views as plan_old_name_migration_views; +use crate::planner_migrations::models::ColumnMigrationInputs; +use crate::planner_migrations::models::MigrationPhase; +use crate::planner_phases::_helpers::inputs::contexts::class_field; +use crate::planner_phases::_helpers::inputs::runtime::runtime_field; +use crate::planner_phases::_helpers::rebuilds::full_refresh::effectively_full_refreshed_model_names; +use crate::planner_phases::models::PlannerWarehouseState; +use crate::planner_warehouse::main::inspection::warehouse_snapshot::warehouse_snapshot; +use crate::planner_warehouse::models::MigrationFingerprintCache; +use crate::planner_warehouse::models::SnapshotRequest; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/mod.rs new file mode 100644 index 0000000000..68672218c6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod inputs; +pub(crate) mod outputs; +pub(crate) mod rebuilds; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs index dfb5acccd7..76c81a61b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs @@ -1,10 +1,10 @@ //! Selected-scope buildability validation before change planning. -use crate::bindings::_helpers::planner_phases::constants::MISSING_UPSTREAM_CODE; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; -use crate::bindings::_helpers::planner_warehouse::source_tables::check_selected_source_tables_exist; +use crate::planner_phases::_helpers::inputs::contexts::class_field; +use crate::planner_phases::_helpers::inputs::runtime::runtime_field; +use crate::planner_phases::_helpers::inputs::runtime::warehouse_field; +use crate::planner_phases::constants::MISSING_UPSTREAM_CODE; +use crate::planner_warehouse::main::sources::check_selected_source_tables_exist::check_selected_source_tables_exist; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message; use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs index c2562c53c8..d0d8f1a8b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs @@ -1,10 +1,10 @@ //! Expected version identities and stale-warning change detection. -use crate::bindings::_helpers::planner_changes::detect::scope_model_changes; -use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::models::IdentityWork; -use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; +use crate::planner_changes::main::scope_model_changes::scope_model_changes; +use crate::planner_identity::main::direct_model_version_identities::direct_model_version_identities; +use crate::planner_phases::_helpers::inputs::contexts::class_field; +use crate::planner_phases::models::IdentityWork; +use crate::planner_phases::models::PlannerIdentityContext; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::sync::PyOnceLock; @@ -217,3 +217,17 @@ pub(crate) fn detect_stale_warning_changes<'py>( ], ) } + +/// `detect_stale_warning_changes_of` as a function of `module`. +pub(crate) fn detect_stale_warning_changes_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(detect_stale_warning_changes_of, module) +} + +/// `planner_identity_work_info` as a function of `module`. +pub(crate) fn planner_identity_work_info_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(planner_identity_work_info, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs new file mode 100644 index 0000000000..fcf2752f38 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod buildability; +pub(crate) mod identities; +pub(crate) mod output_assembly; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs index 14c743ae18..fc26ab100b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs @@ -1,19 +1,21 @@ //! Plan output assembly: storage policies, warnings and direct-mode metadata. -use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; -use crate::bindings::_helpers::planner_entries::plan_output::plan_output_fields; -use crate::bindings::_helpers::planner_identity::staleness::build_stale_out_of_selection_warnings; -use crate::bindings::_helpers::planner_migrations::fingerprint::with_migration_fingerprints; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; -use crate::bindings::_helpers::planner_phases::models::{ - PlanOutputContext, PlanOutputFields, PlanWarningContext, PlannedModelEntries, - StoragePolicyContext, -}; -use crate::bindings::_helpers::planner_phases::retention::{plan_retention, plan_table_types}; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; -use crate::bindings::_helpers::planner_warehouse::source_freshness::source_freshness_unknown_warnings; +use crate::planner_entries::main::plans::plan_output_fields::plan_output_fields; +use crate::planner_entries::models::PlanOutputInputs; +use crate::planner_identity::main::build_stale_out_of_selection_warnings::build_stale_out_of_selection_warnings; +use crate::planner_migrations::main::with_migration_fingerprints::with_migration_fingerprints; +use crate::planner_phases::_helpers::inputs::contexts::class_field; +use crate::planner_phases::_helpers::inputs::runtime::runtime_field; +use crate::planner_phases::_helpers::inputs::runtime::warehouse_field; +use crate::planner_phases::_helpers::rebuilds::retention::plan_retention; +use crate::planner_phases::_helpers::rebuilds::retention::plan_table_types; +use crate::planner_phases::models::PlanOutputContext; +use crate::planner_phases::models::PlanOutputFields; +use crate::planner_phases::models::PlanWarningContext; +use crate::planner_phases::models::PlannedModelEntries; +use crate::planner_phases::models::PlannerIdentityContext; +use crate::planner_phases::models::StoragePolicyContext; +use crate::planner_warehouse::main::sources::source_freshness_unknown_warnings::source_freshness_unknown_warnings; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_scope::constants::MODEL; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/deferred_origins.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/deferred_origins.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/deferred_origins.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/deferred_origins.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs similarity index 86% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs index 6cef425326..b2394aa97c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs @@ -1,18 +1,19 @@ //! Write-hash reconciliation and plan entry construction. -use crate::bindings::_helpers::planner_entries::contexts::default_microbatch_limit_action; -use crate::bindings::_helpers::planner_entries::model_entries::plan_entries; -use crate::bindings::_helpers::planner_entries::models::{ - PlanEntriesRequest, PlanEntryBuildInputs, RelationsRequest, -}; -use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; -use crate::bindings::_helpers::planner_identity::identities::merge_recomputed_model_changes; -use crate::bindings::_helpers::planner_identity::identities::with_honest_model_write_hashes; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; -use crate::bindings::_helpers::planner_phases::models::{ChangeReconciliation, ModelEntryResults}; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; +use crate::planner_entries::main::entries::default_microbatch_limit_action::default_microbatch_limit_action; +use crate::planner_entries::main::entries::plan_entries::plan_entries; +use crate::planner_entries::main::entries::planner_relations_context::planner_relations_context; +use crate::planner_entries::models::PlanEntriesRequest; +use crate::planner_entries::models::PlanEntryBuildInputs; +use crate::planner_entries::models::RelationsRequest; +use crate::planner_identity::main::merge_recomputed_model_changes::merge_recomputed_model_changes; +use crate::planner_identity::main::with_honest_model_write_hashes::with_honest_model_write_hashes; +use crate::planner_phases::_helpers::inputs::contexts::class_field; +use crate::planner_phases::_helpers::inputs::runtime::runtime_field; +use crate::planner_phases::_helpers::inputs::runtime::warehouse_field; +use crate::planner_phases::models::ChangeReconciliation; +use crate::planner_phases::models::ModelEntryResults; +use crate::planner_phases::models::PlannerIdentityContext; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyFrozenSet; use sqlbuild_python_base::boundary::main::keywords::keywords; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/full_refresh.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/full_refresh.rs index ef44a32502..35ac7b4cb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/full_refresh.rs @@ -1,6 +1,6 @@ //! Per-model full-refresh resolution and full-rebuild refusals. -use crate::bindings::_helpers::planner_phases::constants::FULL_REFRESH_DISABLED_REBUILD_CODE; +use crate::planner_phases::constants::FULL_REFRESH_DISABLED_REBUILD_CODE; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::types::PyBool; @@ -96,3 +96,17 @@ pub(crate) fn check_full_rebuild_allowed( ), )) } + +/// `check_full_rebuild_allowed` as a function of `module`. +pub(crate) fn check_full_rebuild_allowed_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(check_full_rebuild_allowed, module) +} + +/// `describe_full_rebuild_cause` as a function of `module`. +pub(crate) fn describe_full_rebuild_cause_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(describe_full_rebuild_cause, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/mod.rs new file mode 100644 index 0000000000..23c86a7a82 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod deferred_origins; +pub(crate) mod execution_changes; +pub(crate) mod full_refresh; +pub(crate) mod retention; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs index 0db0b3022e..ebfa1985dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs @@ -2,12 +2,16 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::planner_phases::constants::{ - BIGQUERY_ADAPTER, DECREASE_WARNING, DOWNGRADE_HISTORY_DAYS, PERMANENT_TABLE_TYPE, - RETENTION_DECREASE, RETENTION_INCREASE, RETENTION_MATCH, SNOWFLAKE_ADAPTER, - TRANSIENT_TABLE_TYPE, -}; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_phases::_helpers::inputs::runtime::runtime_field; +use crate::planner_phases::constants::BIGQUERY_ADAPTER; +use crate::planner_phases::constants::DECREASE_WARNING; +use crate::planner_phases::constants::DOWNGRADE_HISTORY_DAYS; +use crate::planner_phases::constants::PERMANENT_TABLE_TYPE; +use crate::planner_phases::constants::RETENTION_DECREASE; +use crate::planner_phases::constants::RETENTION_INCREASE; +use crate::planner_phases::constants::RETENTION_MATCH; +use crate::planner_phases::constants::SNOWFLAKE_ADAPTER; +use crate::planner_phases::constants::TRANSIENT_TABLE_TYPE; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -654,3 +658,24 @@ pub(crate) fn plan_retention<'py>( } tuple(py, decided) } + +/// `plan_retention` as a function of `module`. +pub(crate) fn plan_retention_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(plan_retention, module) +} + +/// `plan_table_types` as a function of `module`. +pub(crate) fn plan_table_types_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(plan_table_types, module) +} + +/// `table_type_copy_name` as a function of `module`. +pub(crate) fn table_type_copy_name_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(table_type_copy_name, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs new file mode 100644 index 0000000000..e53f254daf --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs @@ -0,0 +1,61 @@ +//! Public constants of planner phases. + +use crate::planner_phases::_helpers::inputs::getters::read_planner_identity_context; +use crate::planner_phases::_helpers::inputs::getters::read_planner_runtime; +use crate::planner_phases::_helpers::inputs::getters::read_planner_scope_resolution; +use crate::planner_phases::_helpers::inputs::getters::read_planner_warehouse_state; +use crate::planner_phases::_helpers::inputs::getters::read_retained_scope; +use crate::planner_phases::models::PlannerIdentityContext; +use crate::planner_phases::models::PlannerRuntime; +use crate::planner_phases::models::PlannerWarehouseState; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; +use sqlbuild_python_project::planner_scope::models::RetainedScope; + +pub(crate) const SNOWFLAKE_ADAPTER: &str = "snowflake"; + +pub(crate) const BIGQUERY_ADAPTER: &str = "bigquery"; + +pub(crate) const TRANSIENT_TABLE_TYPE: &str = "transient"; + +pub(crate) const PERMANENT_TABLE_TYPE: &str = "permanent"; + +pub(crate) const DECREASE_WARNING: &str = + "Retention decreases are irreversible once warehouse history expires."; + +pub(crate) const DOWNGRADE_HISTORY_DAYS: i64 = 90; + +pub(crate) const FULL_REFRESH_DISABLED_REBUILD_CODE: &str = "S203"; + +pub(crate) const MISSING_UPSTREAM_CODE: &str = "S301"; + +pub(crate) const RETENTION_DECREASE: &str = "decrease"; + +pub(crate) const RETENTION_INCREASE: &str = "increase"; + +pub(crate) const RETENTION_MATCH: &str = "match"; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_retained_scope, + }, + GetterEntry { + class: class_of::, + read: read_planner_identity_context, + }, + GetterEntry { + class: class_of::, + read: read_planner_runtime, + }, + GetterEntry { + class: class_of::, + read: read_planner_scope_resolution, + }, + GetterEntry { + class: class_of::, + read: read_planner_warehouse_state, + }, +]; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/class_field.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/class_field.rs new file mode 100644 index 0000000000..ba79b54666 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/class_field.rs @@ -0,0 +1,21 @@ +//! One field of a frozen native planner class passed as a Python object. + +use crate::planner_phases::_helpers::inputs::contexts; +use pyo3::PyClass; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::pyclass::boolean_struct::True; + +/// One field of a frozen native planner class passed as a Python object. +#[inline] +pub fn class_field<'py, T>( + value: &Bound<'py, PyAny>, + field: fn(&T) -> &Py, +) -> PyResult> +where + T: PyClass + Sync, +{ + contexts::class_field(value, field) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/gather_planner_warehouse_state.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/gather_planner_warehouse_state.rs new file mode 100644 index 0000000000..e78e98b11d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/gather_planner_warehouse_state.rs @@ -0,0 +1,17 @@ +//! Gather planner warehouse state. + +use crate::planner_phases::_helpers::inputs::warehouse_state; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Gather planner warehouse state. +#[inline] +pub fn gather_planner_warehouse_state<'py>( + runtime: &Bound<'py, PyAny>, + scopes: &Bound<'py, PyAny>, + overrides: &Bound<'py, PyAny>, + deferral: &Bound<'py, PyAny>, +) -> PyResult> { + warehouse_state::gather_planner_warehouse_state(runtime, scopes, overrides, deferral) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/mod.rs new file mode 100644 index 0000000000..b02e5cc9c9 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/mod.rs @@ -0,0 +1,4 @@ +pub mod class_field; +pub mod gather_planner_warehouse_state; +pub mod runtime_field; +pub mod warehouse_field; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_field.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_field.rs new file mode 100644 index 0000000000..4be5d2937f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_field.rs @@ -0,0 +1,17 @@ +//! One field of a planner runtime passed as a Python object. + +use crate::planner_phases::_helpers::inputs::runtime; +use crate::planner_phases::models::PlannerRuntime; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// One field of a planner runtime passed as a Python object. +#[inline] +pub fn runtime_field<'py>( + runtime: &Bound<'py, PyAny>, + field: fn(&PlannerRuntime) -> &Py, +) -> PyResult> { + runtime::runtime_field(runtime, field) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/warehouse_field.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/warehouse_field.rs new file mode 100644 index 0000000000..5b957c13ff --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/warehouse_field.rs @@ -0,0 +1,17 @@ +//! One field of a planner warehouse state passed as a Python object. + +use crate::planner_phases::_helpers::inputs::runtime; +use crate::planner_phases::models::PlannerWarehouseState; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// One field of a planner warehouse state passed as a Python object. +#[inline] +pub fn warehouse_field<'py>( + warehouse: &Bound<'py, PyAny>, + field: fn(&PlannerWarehouseState) -> &Py, +) -> PyResult> { + runtime::warehouse_field(warehouse, field) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/mod.rs new file mode 100644 index 0000000000..c73411eada --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/mod.rs @@ -0,0 +1,3 @@ +pub mod inputs; +pub mod outputs; +pub mod rebuilds; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_deferred_origin_notes.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_deferred_origin_notes.rs new file mode 100644 index 0000000000..3db676335c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_deferred_origin_notes.rs @@ -0,0 +1,17 @@ +//! Note unselected upstreams deferred to pre-migration names, and each lookup in `failures`. + +use crate::planner_phases::_helpers::outputs::output_assembly; +use crate::planner_phases::models::PlanOutputFields; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Note unselected upstreams deferred to pre-migration names, and each lookup in `failures`. +#[inline] +pub fn add_deferred_origin_notes<'py>( + output: PlanOutputFields<'py>, + (project, scopes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (deferral, failures): (&Bound<'py, PyAny>, &[(String, String)]), +) -> PyResult> { + output_assembly::add_deferred_origin_notes(output, (project, scopes), (deferral, failures)) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_metadata.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_metadata.rs new file mode 100644 index 0000000000..eaa8972aa0 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_metadata.rs @@ -0,0 +1,17 @@ +//! Add the direct-mode source freshness, remaining stale models and selection diagnostics metadata. + +use crate::planner_phases::_helpers::outputs::output_assembly; +use crate::planner_phases::models::PlanOutputFields; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Add the direct-mode source freshness, remaining stale models and selection diagnostics metadata. +#[inline] +pub fn add_plan_metadata<'py>( + output: PlanOutputFields<'py>, + source_freshness: &Bound<'py, PyAny>, + policies: &Bound<'py, PyAny>, +) -> PyResult> { + output_assembly::add_plan_metadata(output, source_freshness, policies) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_warnings.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_warnings.rs new file mode 100644 index 0000000000..6e42a48d1d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_warnings.rs @@ -0,0 +1,32 @@ +//! Add unknown source freshness warnings and, with selection diagnostics, stale-selection warnings. + +use crate::planner_phases::_helpers::outputs::output_assembly; +use crate::planner_phases::models::PlanOutputFields; +use crate::planner_phases::models::PlanWarningContext; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Add unknown source freshness warnings and, with selection diagnostics, stale-selection warnings. +#[inline] +pub fn add_plan_warnings<'py>( + output: PlanOutputFields<'py>, + PlanWarningContext { + scopes, + warehouse, + identities, + stale_warning_changes, + }: PlanWarningContext<'_, 'py>, + (source_freshness, policies): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + output_assembly::add_plan_warnings( + output, + PlanWarningContext { + scopes, + warehouse, + identities, + stale_warning_changes, + }, + (source_freshness, policies), + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_storage_policies.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_storage_policies.rs new file mode 100644 index 0000000000..212eae9c22 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_storage_policies.rs @@ -0,0 +1,28 @@ +//! Add migration fingerprints, rename hints, migration, table-type and retention entries. + +use crate::planner_phases::_helpers::outputs::output_assembly; +use crate::planner_phases::models::PlanOutputFields; +use crate::planner_phases::models::StoragePolicyContext; +use pyo3::prelude::PyResult; + +/// Add migration fingerprints, rename hints, migration, table-type and retention entries. +#[inline] +pub fn add_storage_policies<'py>( + output: PlanOutputFields<'py>, + StoragePolicyContext { + runtime, + warehouse, + scopes, + policies, + }: StoragePolicyContext<'_, 'py>, +) -> PyResult> { + output_assembly::add_storage_policies( + output, + StoragePolicyContext { + runtime, + warehouse, + scopes, + policies, + }, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/base_plan_output.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/base_plan_output.rs new file mode 100644 index 0000000000..3db2cd82d9 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/base_plan_output.rs @@ -0,0 +1,43 @@ +//! The plan output's base fields from the planned entries, relations and source freshness. + +use crate::planner_phases::_helpers::outputs::output_assembly; +use crate::planner_phases::models::PlanOutputContext; +use crate::planner_phases::models::PlanOutputFields; +use crate::planner_phases::models::PlannedModelEntries; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The plan output's base fields from the planned entries, relations and source freshness. +#[inline] +pub fn base_plan_output<'py>( + PlanOutputContext { + runtime, + warehouse, + identities, + overrides, + }: PlanOutputContext<'_, 'py>, + PlannedModelEntries { + execution_scope, + reconciliation, + entries, + source_freshness, + }: PlannedModelEntries<'_, 'py>, + planned_sql_tests: &Bound<'py, PyAny>, +) -> PyResult> { + output_assembly::base_plan_output( + PlanOutputContext { + runtime, + warehouse, + identities, + overrides, + }, + PlannedModelEntries { + execution_scope, + reconciliation, + entries, + source_freshness, + }, + planned_sql_tests, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/build_planner_identity_context.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/build_planner_identity_context.rs new file mode 100644 index 0000000000..dd98583df5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/build_planner_identity_context.rs @@ -0,0 +1,20 @@ +//! Build planner identity context. + +use crate::planner_phases::_helpers::outputs::identities; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Build planner identity context. +#[inline] +pub fn build_planner_identity_context<'py>( + project: &Bound<'py, PyAny>, + (scopes, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + include_stale_warning_identities: bool, +) -> PyResult> { + identities::build_planner_identity_context( + project, + (scopes, snapshot), + include_stale_warning_identities, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/check_selected_scope_buildability.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/check_selected_scope_buildability.rs new file mode 100644 index 0000000000..db8be8e2b6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/check_selected_scope_buildability.rs @@ -0,0 +1,17 @@ +//! Check selected scope buildability. + +use crate::planner_phases::_helpers::outputs::buildability; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Check selected scope buildability. +#[inline] +pub fn check_selected_scope_buildability<'py>( + runtime: &Bound<'py, PyAny>, + scopes: &Bound<'py, PyAny>, + warehouse: &Bound<'py, PyAny>, + deferral: &Bound<'py, PyAny>, +) -> PyResult<()> { + buildability::check_selected_scope_buildability(runtime, scopes, warehouse, deferral) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes.rs new file mode 100644 index 0000000000..f63cf08d39 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes.rs @@ -0,0 +1,20 @@ +//! Changes of stale-warning models whose identities differ from the execution identities. + +use crate::planner_phases::_helpers::outputs::identities; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Changes of stale-warning models whose identities differ from the execution identities. +#[inline] +pub fn detect_stale_warning_changes<'py>( + (project, scopes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + snapshot: &Bound<'py, PyAny>, + (identities, execution_changes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + identities::detect_stale_warning_changes( + (project, scopes), + snapshot, + (identities, execution_changes), + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes_of_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes_of_function.rs new file mode 100644 index 0000000000..a7bea34508 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes_of_function.rs @@ -0,0 +1,11 @@ +//! `detect_stale_warning_changes_of` as a function of `module`. + +use crate::planner_phases::_helpers::outputs::identities; + +/// `detect_stale_warning_changes_of` as a function of `module`. +#[inline] +pub fn detect_stale_warning_changes_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + identities::detect_stale_warning_changes_of_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs new file mode 100644 index 0000000000..69b6f2dbe9 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs @@ -0,0 +1,11 @@ +pub mod add_deferred_origin_notes; +pub mod add_plan_metadata; +pub mod add_plan_warnings; +pub mod add_storage_policies; +pub mod base_plan_output; +pub mod build_planner_identity_context; +pub mod check_selected_scope_buildability; +pub mod detect_stale_warning_changes; +pub mod detect_stale_warning_changes_of_function; +pub mod planner_identity_work_info_function; +pub mod unselected_upstream_models; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/planner_identity_work_info_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/planner_identity_work_info_function.rs new file mode 100644 index 0000000000..052eecee81 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/planner_identity_work_info_function.rs @@ -0,0 +1,11 @@ +//! `planner_identity_work_info` as a function of `module`. + +use crate::planner_phases::_helpers::outputs::identities; + +/// `planner_identity_work_info` as a function of `module`. +#[inline] +pub fn planner_identity_work_info_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + identities::planner_identity_work_info_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/unselected_upstream_models.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/unselected_upstream_models.rs new file mode 100644 index 0000000000..3c38fd5cda --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/unselected_upstream_models.rs @@ -0,0 +1,12 @@ +//! The unselected models the selected scope reads, sorted. + +use crate::planner_phases::_helpers::outputs::output_assembly; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The unselected models the selected scope reads, sorted. +#[inline] +pub fn unselected_upstream_models(scopes: &Bound<'_, PyAny>) -> PyResult> { + output_assembly::unselected_upstream_models(scopes) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/build_planner_entry_results.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/build_planner_entry_results.rs new file mode 100644 index 0000000000..28a4974f80 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/build_planner_entry_results.rs @@ -0,0 +1,24 @@ +//! Model plan entries for the pruned execution scope, with the freshness and override inputs they need. + +use crate::planner_phases::_helpers::rebuilds::execution_changes; +use crate::planner_phases::models::ChangeReconciliation; +use crate::planner_phases::models::ModelEntryResults; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Model plan entries for the pruned execution scope, with the freshness and override inputs they need. +#[inline] +pub fn build_planner_entry_results<'py>( + (runtime, warehouse): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (overrides, deferral): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (execution_scope, reconciliation): (&Bound<'py, PyAny>, &ChangeReconciliation<'py>), + source_freshness: &Bound<'py, PyAny>, +) -> PyResult> { + execution_changes::build_planner_entry_results( + (runtime, warehouse), + (overrides, deferral), + (execution_scope, reconciliation), + source_freshness, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed.rs new file mode 100644 index 0000000000..0148cb6de6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed.rs @@ -0,0 +1,16 @@ +//! Check full rebuild allowed. + +use crate::planner_phases::_helpers::rebuilds::full_refresh; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Check full rebuild allowed. +#[inline] +pub fn check_full_rebuild_allowed( + model: &Bound<'_, PyAny>, + change_result: &Bound<'_, PyAny>, + full_rebuild_cause: &str, +) -> PyResult<()> { + full_refresh::check_full_rebuild_allowed(model, change_result, full_rebuild_cause) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed_function.rs new file mode 100644 index 0000000000..e3ce35309f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed_function.rs @@ -0,0 +1,11 @@ +//! `check_full_rebuild_allowed` as a function of `module`. + +use crate::planner_phases::_helpers::rebuilds::full_refresh; + +/// `check_full_rebuild_allowed` as a function of `module`. +#[inline] +pub fn check_full_rebuild_allowed_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + full_refresh::check_full_rebuild_allowed_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/defer_to_migration_origins.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/defer_to_migration_origins.rs new file mode 100644 index 0000000000..bc0aa22705 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/defer_to_migration_origins.rs @@ -0,0 +1,21 @@ +//! Redirect unselected `models` unbuilt in the deferred target to their `migrate_from` relation. + +use crate::planner_phases::_helpers::rebuilds::deferred_origins; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Redirect unselected `models` unbuilt in the deferred target to their `migrate_from` relation. +#[inline] +pub fn defer_to_migration_origins<'py>( + (project, adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + (locations, relations): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), + models: &[String], +) -> PyResult> { + deferred_origins::defer_to_migration_origins( + (project, adapter, connection), + (locations, relations), + models, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause.rs new file mode 100644 index 0000000000..ec147f0570 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause.rs @@ -0,0 +1,15 @@ +//! Describe full rebuild cause. + +use crate::planner_phases::_helpers::rebuilds::full_refresh; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Describe full rebuild cause. +#[inline] +pub fn describe_full_rebuild_cause( + change_result: &Bound<'_, PyAny>, + full_refresh: bool, +) -> PyResult { + full_refresh::describe_full_rebuild_cause(change_result, full_refresh) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause_function.rs new file mode 100644 index 0000000000..5a5a37bf68 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause_function.rs @@ -0,0 +1,11 @@ +//! `describe_full_rebuild_cause` as a function of `module`. + +use crate::planner_phases::_helpers::rebuilds::full_refresh; + +/// `describe_full_rebuild_cause` as a function of `module`. +#[inline] +pub fn describe_full_rebuild_cause_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + full_refresh::describe_full_rebuild_cause_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effective_target_config.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effective_target_config.rs new file mode 100644 index 0000000000..0745520645 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effective_target_config.rs @@ -0,0 +1,12 @@ +//! The resolved config of the planned target, or defaults without one. + +use crate::planner_phases::_helpers::rebuilds::retention; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The resolved config of the planned target, or defaults without one. +#[inline] +pub fn effective_target_config<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { + retention::effective_target_config(runtime) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs new file mode 100644 index 0000000000..93ea4b0e66 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs @@ -0,0 +1,15 @@ +//! Names of models effectively full refreshed for this invocation. + +use crate::planner_phases::_helpers::rebuilds::full_refresh; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Names of models effectively full refreshed for this invocation. +#[inline] +pub fn effectively_full_refreshed_model_names<'py>( + project: &Bound<'py, PyAny>, + cli_full_refresh: bool, +) -> PyResult> { + full_refresh::effectively_full_refreshed_model_names(project, cli_full_refresh) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/mod.rs new file mode 100644 index 0000000000..b3c1c34f26 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/mod.rs @@ -0,0 +1,13 @@ +pub mod build_planner_entry_results; +pub mod check_full_rebuild_allowed; +pub mod check_full_rebuild_allowed_function; +pub mod defer_to_migration_origins; +pub mod describe_full_rebuild_cause; +pub mod describe_full_rebuild_cause_function; +pub mod effective_target_config; +pub mod effectively_full_refreshed_model_names; +pub mod plan_retention_function; +pub mod plan_table_types_function; +pub mod reconcile_execution_changes; +pub mod resolve_model_full_refresh; +pub mod table_type_copy_name_function; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_retention_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_retention_function.rs new file mode 100644 index 0000000000..d2bcbeb000 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_retention_function.rs @@ -0,0 +1,11 @@ +//! `plan_retention` as a function of `module`. + +use crate::planner_phases::_helpers::rebuilds::retention; + +/// `plan_retention` as a function of `module`. +#[inline] +pub fn plan_retention_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + retention::plan_retention_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_table_types_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_table_types_function.rs new file mode 100644 index 0000000000..d239a14705 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_table_types_function.rs @@ -0,0 +1,11 @@ +//! `plan_table_types` as a function of `module`. + +use crate::planner_phases::_helpers::rebuilds::retention; + +/// `plan_table_types` as a function of `module`. +#[inline] +pub fn plan_table_types_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + retention::plan_table_types_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/reconcile_execution_changes.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/reconcile_execution_changes.rs new file mode 100644 index 0000000000..95b3dee371 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/reconcile_execution_changes.rs @@ -0,0 +1,23 @@ +//! The execution scope's changes with honest write hashes, and the actions merged from them. + +use crate::planner_phases::_helpers::rebuilds::execution_changes; +use crate::planner_phases::models::ChangeReconciliation; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The execution scope's changes with honest write hashes, and the actions merged from them. +#[inline] +pub fn reconcile_execution_changes<'py>( + (execution_scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + identities: &Bound<'py, PyAny>, + resolved_actions: &Bound<'py, PyAny>, + changes: &Bound<'py, PyAny>, +) -> PyResult> { + execution_changes::reconcile_execution_changes( + (execution_scope, snapshot), + identities, + resolved_actions, + changes, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/resolve_model_full_refresh.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/resolve_model_full_refresh.rs new file mode 100644 index 0000000000..0b197a08b7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/resolve_model_full_refresh.rs @@ -0,0 +1,15 @@ +//! dbt-compatible nullable model full-refresh semantics. + +use crate::planner_phases::_helpers::rebuilds::full_refresh; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// dbt-compatible nullable model full-refresh semantics. +#[inline] +pub fn resolve_model_full_refresh( + model: &Bound<'_, PyAny>, + cli_full_refresh: bool, +) -> PyResult { + full_refresh::resolve_model_full_refresh(model, cli_full_refresh) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/table_type_copy_name_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/table_type_copy_name_function.rs new file mode 100644 index 0000000000..f58f400825 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/table_type_copy_name_function.rs @@ -0,0 +1,11 @@ +//! `table_type_copy_name` as a function of `module`. + +use crate::planner_phases::_helpers::rebuilds::retention; + +/// `table_type_copy_name` as a function of `module`. +#[inline] +pub fn table_type_copy_name_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + retention::table_type_copy_name_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/mod.rs new file mode 100644 index 0000000000..94b56e8614 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/mod.rs @@ -0,0 +1,6 @@ +//! Native execution-planning phases over Python planner models and adapter callbacks. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/models.rs b/crates/sqlbuild-python-planner/src/planner_phases/models.rs new file mode 100644 index 0000000000..ec83416021 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/models.rs @@ -0,0 +1,108 @@ +//! Public records of planner phases. + +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::pyclass; +use std::collections::BTreeMap; + +/// The plan output's fields and metadata entries, each set before the output is built. +#[derive(Debug)] +pub struct PlanOutputFields<'py> { + pub fields: Vec<(&'static str, Bound<'py, PyAny>)>, + pub metadata: Vec<(&'static str, Bound<'py, PyAny>)>, +} + +/// The model changes and resolved actions after write-hash reconciliation. +#[derive(Debug)] +pub struct ChangeReconciliation<'py> { + pub changes: Bound<'py, PyAny>, + pub resolved_actions: Bound<'py, PyAny>, +} + +/// Planned model entries and the warnings planning them raised. +#[derive(Debug)] +pub struct ModelEntryResults<'py> { + pub entries: Vec>, + pub warnings: Vec>, +} + +/// The runtime, warehouse state, identities and overrides the base plan output reads. +#[derive(Debug)] +pub struct PlanOutputContext<'a, 'py> { + pub runtime: &'a Bound<'py, PyAny>, + pub warehouse: &'a Bound<'py, PyAny>, + pub identities: &'a Bound<'py, PyAny>, + pub overrides: &'a Bound<'py, PyAny>, +} + +/// The execution scope, reconciled changes, planned entries and source freshness of a plan. +#[derive(Debug)] +pub struct PlannedModelEntries<'a, 'py> { + pub execution_scope: &'a Bound<'py, PyAny>, + pub(crate) reconciliation: &'a ChangeReconciliation<'py>, + pub entries: ModelEntryResults<'py>, + pub source_freshness: &'a Bound<'py, PyAny>, +} + +/// The runtime, warehouse state, scopes and policies storage policy planning reads. +#[derive(Debug)] +pub struct StoragePolicyContext<'a, 'py> { + pub runtime: &'a Bound<'py, PyAny>, + pub warehouse: &'a Bound<'py, PyAny>, + pub scopes: &'a Bound<'py, PyAny>, + pub policies: &'a Bound<'py, PyAny>, +} + +/// The scopes, warehouse state, identities and changes stale-selection warnings read. +#[derive(Debug)] +pub struct PlanWarningContext<'a, 'py> { + pub scopes: &'a Bound<'py, PyAny>, + pub warehouse: &'a Bound<'py, PyAny>, + pub identities: &'a Bound<'py, PyAny>, + pub stale_warning_changes: &'a Bound<'py, PyAny>, +} + +/// Resolved execution environment shared by planner phases. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct PlannerRuntime { + pub project: Py, + pub adapter: Py, + pub connection: Py, + pub project_config: Py, + pub local_config: Py, + pub on_progress: Py, + pub invocation_time: Py, +} + +/// Warehouse snapshot, inspection relations and migration plans gathered once per plan. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct PlannerWarehouseState { + pub snapshot: Py, + pub inspection_relations: Py, + pub migration_entries: Py, + pub migration_warnings: Py, + pub column_migration_entries: Py, + pub column_rename_hints: Py, + pub old_name_view_entries: Py, + pub old_name_views: Py, + pub migration_fingerprints: Py, +} + +/// Expected version identities for inspection and stale-warning scopes. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct PlannerIdentityContext { + pub version_identities: Py, + pub stale_warning_identities: Py, +} + +/// Planner identity work since the last reset, for work-count tests. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct IdentityWork { + pub identity_builds: u64, + pub stale_detections: BTreeMap, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_bounds.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_bounds.rs index 4c686ff2ed..489e89dc70 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_bounds.rs @@ -2,8 +2,8 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::planner_resolution::maximum_safety::apply_maximum_start_policy; -use crate::bindings::_helpers::planner_resolution::values::string; +use crate::planner_resolution::_helpers::references::values::string; +use crate::planner_resolution::_helpers::safety::maximum_safety::apply_maximum_start_policy; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; @@ -24,7 +24,7 @@ use sqlbuild_python_front::cursor::main::conversion::algebra_error::algebra_erro use sqlbuild_python_front::cursor::main::scalars::try_parse::try_parse; use sqlbuild_python_front::cursor::models::CursorSession; -pub(super) fn optional_scalar<'py>( +pub(crate) fn optional_scalar<'py>( session: &CursorSession<'py>, value: &Bound<'py, PyAny>, ) -> PyResult> { @@ -35,7 +35,7 @@ pub(super) fn optional_scalar<'py>( } } -pub(super) fn scalars<'py>( +pub(crate) fn scalars<'py>( session: &CursorSession<'py>, object: &Bound<'py, PyAny>, field: &str, @@ -71,7 +71,7 @@ fn read_snapshot<'py>( }) } -pub(super) fn duration(value: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn duration(value: &Bound<'_, PyAny>) -> PyResult> { if value.is_none() { return Ok(None); } @@ -159,7 +159,7 @@ pub(crate) fn apply_typed_cursor_replay_policy<'py>( session.python_scalar(&result) } -pub(super) fn required_scalar<'py>( +pub(crate) fn required_scalar<'py>( session: &CursorSession<'py>, raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, @@ -300,7 +300,7 @@ pub(crate) fn compute_cursor_bounds<'py>( apply_maximum_start_policy(py, Some(keywords(py, &policy_args)?)) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(compute_cursor_bounds, module)?)?; module.add_function(wrap_pyfunction!(apply_typed_cursor_replay_policy, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs index 5cbd26a115..a5ba16028f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs @@ -2,22 +2,21 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; -use crate::bindings::_helpers::plan_output::display::resolve_bounded_cursor_override; -use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_resolution::constants::CURSOR_START; -use crate::bindings::_helpers::planner_resolution::cursor_bounds::compute_cursor_bounds; -use crate::bindings::_helpers::planner_resolution::cursor_policies::{ - future_cursor_config, start_cursor_config, -}; -use crate::bindings::_helpers::planner_resolution::cursor_windows::cursor_inputs_without_rows; -use crate::bindings::_helpers::planner_resolution::cursor_windows::empty_cursor_window; -use crate::bindings::_helpers::planner_resolution::cursor_windows::empty_input_rebuild_refusal; -use crate::bindings::_helpers::planner_resolution::cursor_windows::unreadable_cursor_inputs_error; -use crate::bindings::_helpers::planner_resolution::cursor_windows::without_destination_cursor; -use crate::bindings::_helpers::planner_resolution::future_safety::resolve_future_cursor_safety; -use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest; -use crate::bindings::_helpers::planner_resolution::values::string; +use crate::plan_output::main::resolve_bounded_cursor_override::resolve_bounded_cursor_override; +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_phases::main::inputs::class_field::class_field; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::compute_cursor_bounds; +use crate::planner_resolution::_helpers::cursor::cursor_policies::future_cursor_config; +use crate::planner_resolution::_helpers::cursor::cursor_policies::start_cursor_config; +use crate::planner_resolution::_helpers::cursor::cursor_windows::cursor_inputs_without_rows; +use crate::planner_resolution::_helpers::cursor::cursor_windows::empty_cursor_window; +use crate::planner_resolution::_helpers::cursor::cursor_windows::empty_input_rebuild_refusal; +use crate::planner_resolution::_helpers::cursor::cursor_windows::unreadable_cursor_inputs_error; +use crate::planner_resolution::_helpers::cursor::cursor_windows::without_destination_cursor; +use crate::planner_resolution::_helpers::references::values::string; +use crate::planner_resolution::_helpers::safety::future_safety::resolve_future_cursor_safety; +use crate::planner_resolution::constants::CURSOR_START; +use crate::planner_resolution::models::ModelSqlRequest; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::{IntoPyObject, PyErr}; use sqlbuild_python_base::boundary::main::call_with::call_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs index b5d65ccb4c..67f36b7065 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs @@ -1,6 +1,7 @@ //! Selected-model inspection and untyped override conversion. -use crate::bindings::_helpers::planner_resolution::values::{method, string}; +use crate::planner_resolution::_helpers::references::values::method; +use crate::planner_resolution::_helpers::references::values::string; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -96,6 +97,6 @@ pub(crate) fn type_cursor_overrides<'py>( method(overrides, "typed_as", &[("cursor_type", kind)]) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(type_cursor_overrides_of, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_policies.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_policies.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs index 605b995324..56a49d21cb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs @@ -1,24 +1,25 @@ //! Plan-entry cursor windows: overrides, planner-owned bounds and microbatch ranges. +use crate::planner_resolution::models::CursorWindowInputs; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; -use crate::bindings::_helpers::plan_output::display::resolve_bounded_cursor_override; -use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_resolution::cursor_bounds::compute_cursor_bounds; -use crate::bindings::_helpers::planner_resolution::cursor_bounds::duration; -use crate::bindings::_helpers::planner_resolution::cursor_policies::{ - future_cursor_config, start_cursor_config, -}; -use crate::bindings::_helpers::planner_resolution::cursor_safety::{replace, utc_clock}; -use crate::bindings::_helpers::planner_resolution::cursor_windows::cursor_inputs_without_rows; -use crate::bindings::_helpers::planner_resolution::cursor_windows::normalize_cursor_snapshot_grain; -use crate::bindings::_helpers::planner_resolution::cursor_windows::resolve_effective_timestamp_grain; -use crate::bindings::_helpers::planner_resolution::cursor_windows::without_destination_cursor; -use crate::bindings::_helpers::planner_resolution::future_safety::resolve_future_cursor_safety; -use crate::bindings::_helpers::planner_resolution::values::string; -use crate::bindings::_helpers::planner_strategy::model_entries::resolve_microbatch_lookback; +use crate::plan_output::main::resolve_bounded_cursor_override::resolve_bounded_cursor_override; +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_phases::main::inputs::class_field::class_field; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::compute_cursor_bounds; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::duration; +use crate::planner_resolution::_helpers::cursor::cursor_policies::future_cursor_config; +use crate::planner_resolution::_helpers::cursor::cursor_policies::start_cursor_config; +use crate::planner_resolution::_helpers::cursor::cursor_safety::replace; +use crate::planner_resolution::_helpers::cursor::cursor_safety::utc_clock; +use crate::planner_resolution::_helpers::cursor::cursor_windows::cursor_inputs_without_rows; +use crate::planner_resolution::_helpers::cursor::cursor_windows::normalize_cursor_snapshot_grain; +use crate::planner_resolution::_helpers::cursor::cursor_windows::resolve_effective_timestamp_grain; +use crate::planner_resolution::_helpers::cursor::cursor_windows::without_destination_cursor; +use crate::planner_resolution::_helpers::references::values::string; +use crate::planner_resolution::_helpers::safety::future_safety::resolve_future_cursor_safety; +use crate::planner_strategy::main::model_entries::resolve_microbatch_lookback::resolve_microbatch_lookback; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyString, PyTuple}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; @@ -72,19 +73,6 @@ const WINDOW_OPTIONAL: [&str; 5] = [ "cursor_input_relations", ]; -/// The planner inputs that decide one model's plan-time cursor window. -pub(crate) struct CursorWindowInputs<'py> { - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) backfill: Bound<'py, PyAny>, - pub(crate) overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), - pub(crate) runtime_owned: bool, - pub(crate) policies: (Bound<'py, PyAny>, Bound<'py, PyAny>), - pub(crate) invocation_time: Bound<'py, PyAny>, - pub(crate) full_refresh: bool, - pub(crate) replaces_relation: bool, - pub(crate) cursor_input_relations: Bound<'py, PyAny>, -} - /// One string-valued model configuration entry. pub(crate) fn config_str<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { let py: Python<'py> = model.py(); @@ -662,7 +650,7 @@ fn resolve_cursor_overrides<'py>( ) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(compute_plan_cursor_bounds, module)?)?; module.add_function(wrap_pyfunction!(compute_microbatch_range, module)?)?; module.add_function(wrap_pyfunction!(resolve_cursor_overrides, module)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_safety.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_safety.rs index 052858c4ef..61a5fa8cca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_safety.rs @@ -1,6 +1,6 @@ //! Shared safety horizon conversion and Python evidence construction. -use crate::bindings::_helpers::planner_resolution::cursor_bounds::duration; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::duration; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use sqlbuild_cursor::algebra::models::{DurationSpan, Grain, Scalar}; use sqlbuild_cursor::planner::main::safety_horizon::safety_horizon; @@ -16,11 +16,11 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cursor::models::CursorSession; -pub(super) fn safety_error(py: Python<'_>, class: ErrorId, message: &str) -> PyErr { +pub(crate) fn safety_error(py: Python<'_>, class: ErrorId, message: &str) -> PyErr { new_plain_error(py, class, message) } -pub(super) fn safety_duration(value: &Bound<'_, PyAny>, future: bool) -> PyResult { +pub(crate) fn safety_duration(value: &Bound<'_, PyAny>, future: bool) -> PyResult { if let Some(parsed) = duration(value)? { return Ok(parsed); } @@ -45,17 +45,17 @@ pub(super) fn safety_duration(value: &Bound<'_, PyAny>, future: bool) -> PyResul )) } -pub(super) fn timestamp_bound(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn timestamp_bound(value: &Bound<'_, PyAny>) -> PyResult { let py = value.py(); Ok(value.is_instance(&record_class(py, RecordId::DateValue)?)? || value.is_instance(&record_class(py, RecordId::TimestampValue)?)?) } -pub(super) fn utc_clock<'py>(clock: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn utc_clock<'py>(clock: &Bound<'py, PyAny>) -> PyResult> { clock.call_method1("astimezone", (imported(clock.py(), "datetime", "UTC")?,)) } -pub(super) fn horizon<'py>( +pub(crate) fn horizon<'py>( session: &CursorSession<'py>, discovered: &Bound<'py, PyAny>, clock_and_grain: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -85,7 +85,7 @@ pub(super) fn horizon<'py>( )) } -pub(super) fn replace<'py>( +pub(crate) fn replace<'py>( object: &Bound<'py, PyAny>, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_warnings.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_warnings.rs index 599890d6d9..6dfd368657 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_warnings.rs @@ -1,6 +1,6 @@ //! Prominent warnings for applied future-end and automatic-start cursor safety. -use crate::bindings::_helpers::planner_resolution::values::string; +use crate::planner_resolution::_helpers::references::values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -111,7 +111,7 @@ fn maximum_start_cap_warning(bounds: &Bound<'_, PyAny>) -> PyResult) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(future_cursor_cap_warning, module)?)?; module.add_function(wrap_pyfunction!(maximum_start_cap_warning, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_windows.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_windows.rs index 46dcb20ea1..c8bf37def0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_windows.rs @@ -1,6 +1,6 @@ //! Empty-window decisions and snapshot grain conversion. -use crate::bindings::_helpers::planner_resolution::values::string; +use crate::planner_resolution::_helpers::references::values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; @@ -262,7 +262,7 @@ pub(crate) fn normalize_cursor_snapshot_grain<'py>( replaced_keywords(cursor_snapshot, &kwargs) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(decide_empty_cursor_inputs, module)?)?; module.add_function(wrap_pyfunction!(empty_cursor_window, module)?)?; module.add_function(wrap_pyfunction!(cursor_window_override_flags, module)?)?; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/mod.rs new file mode 100644 index 0000000000..6f046db60c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod cursor_bounds; +pub(crate) mod cursor_model; +pub(crate) mod cursor_overrides; +pub(crate) mod cursor_policies; +pub(crate) mod cursor_ranges; +pub(crate) mod cursor_safety; +pub(crate) mod cursor_warnings; +pub(crate) mod cursor_windows; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/mod.rs new file mode 100644 index 0000000000..22f80fcf37 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod cursor; +pub(crate) mod references; +pub(crate) mod safety; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs index 01961f6424..f913221385 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs @@ -34,7 +34,7 @@ pub(crate) fn build_function_locations<'py>( locations(functions) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(build_model_locations, module)?)?; module.add_function(wrap_pyfunction!(build_seed_locations, module)?)?; module.add_function(wrap_pyfunction!(build_function_locations, module)?) diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/mod.rs new file mode 100644 index 0000000000..6f4a003b08 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod locations; +pub(crate) mod orchestration; +pub(crate) mod refs; +pub(crate) mod sources; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs similarity index 66% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs index 453f203e19..fe2394b45f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs @@ -1,30 +1,20 @@ //! Model, function and SQL-hook orchestration through adapter-owned rendering. -use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_resolution::cursor_model::{ - compute_model_cursor_bounds, unbounded, -}; -use crate::bindings::_helpers::planner_resolution::models::{CursorFiltering, ModelSqlRequest}; -use crate::bindings::_helpers::planner_resolution::refs::{ - resolve_dbt_ref_references, resolve_refs, resolve_table_function_references, - resolve_udf_references, -}; -use crate::bindings::_helpers::planner_resolution::sources::resolve_sources; +use crate::planner_resolution::_helpers::cursor::cursor_model::compute_model_cursor_bounds; +use crate::planner_resolution::_helpers::cursor::cursor_model::unbounded; +use crate::planner_resolution::models::CursorFiltering; +use crate::planner_resolution::models::ModelSqlRequest; +use crate::planner_resolution::models::Resolution; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{PyErr, pyfunction, wrap_pyfunction}; use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; -use sqlbuild_python_front::attachments::main::assert_no_unresolved_sql_markers::assert_no_unresolved_sql_markers; use sqlbuild_python_front::attachments::main::resolve_cursor_intrinsics::resolve_cursor_intrinsics; use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; @@ -39,111 +29,6 @@ use sqlbuild_python_project::resource_views::main::function_reads::body_sql::bod const APPEND_STRATEGY: &str = "append"; -/// What resolving one model's, hook's or function's references reads. -pub(crate) struct Resolution<'py> { - pub(crate) adapter: Bound<'py, PyAny>, - external: Bound<'py, PyAny>, - source_map: Bound<'py, PyAny>, - source_warehouse_columns: Bound<'py, PyAny>, - model_locations: Bound<'py, PyAny>, - seed_locations: Bound<'py, PyAny>, - function_locations: Bound<'py, PyAny>, -} - -impl<'py> Resolution<'py> { - /// Resolution against a `ModelPlanContext`'s relations. - pub(crate) fn of_context( - adapter: Bound<'py, PyAny>, - context: &Bound<'py, PyAny>, - external: Bound<'py, PyAny>, - ) -> PyResult { - Ok(Self { - adapter, - external, - source_map: class_field(context, |c: &ModelPlanContext| &c.source_map)?, - source_warehouse_columns: class_field(context, |c: &ModelPlanContext| { - &c.source_warehouse_columns - })?, - model_locations: class_field(context, |c: &ModelPlanContext| &c.model_locations)?, - seed_locations: class_field(context, |c: &ModelPlanContext| &c.seed_locations)?, - function_locations: class_field(context, |c: &ModelPlanContext| &c.function_locations)?, - }) - } - - fn resolve(&self, sql: &str, filtering: &CursorFiltering<'py>) -> PyResult { - let py = self.adapter.py(); - let resolved: String = resolve_sources( - sql, - (&self.source_map, &self.source_warehouse_columns), - &self.adapter, - filtering, - )?; - let resolved: String = resolve_refs( - &resolved, - (&self.model_locations, &self.seed_locations), - &self.adapter, - filtering, - )?; - let resolved: String = resolve_dbt_ref_references(&resolved, Some(&self.external))?; - let locations = if self.function_locations.is_truthy()? { - self.function_locations.clone() - } else { - PyDict::new(py).into_any() - }; - let resolved: String = resolve_udf_references(&resolved, &locations, &self.adapter)?; - resolve_table_function_references(&resolved, &locations, &self.adapter) - } - - fn assert_resolved(&self, sql: &str, label: &str) -> PyResult<()> { - let py = self.adapter.py(); - assert_no_unresolved_sql_markers( - py, - &PyString::new(py, sql), - &format!("{label} planned SQL"), - ) - } - - fn unbounded(&self, sql: Bound<'py, PyAny>, label: &str) -> PyResult { - let py = self.adapter.py(); - let resolved = self.resolve( - &sql.extract::()?, - &CursorFiltering { - bounds: py.None().into_bound(py), - filter_inputs: PyDict::new(py).into_any(), - cursor_type: py.None().into_bound(py), - lower_bound_inclusive: true, - }, - )?; - self.assert_resolved(&resolved, label)?; - Ok(resolved) - } - - fn hook(&self, value: Bound<'py, PyAny>, label: &str) -> PyResult> { - let py = value.py(); - if value.is_instance(&record_class(py, RecordId::SqlHookEntry)?)? { - let kwargs = PyDict::new(py); - kwargs.set_item( - "statement", - self.unbounded(native_field(&value, "statement")?, label)?, - )?; - return replaced_keywords(&value, &kwargs); - } - let is_list = value.is_instance_of::(); - if is_list || value.is_instance_of::() { - let mut resolved: Vec> = Vec::new(); - for (index, item) in value.try_iter()?.enumerate() { - resolved.push(self.hook(item?, &format!("{label}[{index}]"))?); - } - return if is_list { - Ok(PyList::new(py, resolved)?.into_any()) - } else { - Ok(PyTuple::new(py, resolved)?.into_any()) - }; - } - Ok(value) - } -} - #[pyfunction(name = "resolve_model_sql")] #[pyo3(signature = (**kwargs))] fn resolve_model_sql_of<'py>( @@ -366,7 +251,7 @@ pub(crate) fn model_hook_entries<'py>( PyTuple::new(model.py(), hooks) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(resolve_model_sql_of, module)?)?; module.add_function(wrap_pyfunction!(resolve_function_sql, module)?)?; module.add_function(wrap_pyfunction!(resolve_model_hook_entries, module)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs index 8429bcf5ba..a17cd50ced 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs @@ -2,10 +2,11 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; -use crate::bindings::_helpers::planner_resolution::values::{ - cursor_wrap, filtering_of, method, string, -}; +use crate::planner_resolution::_helpers::references::values::cursor_wrap; +use crate::planner_resolution::_helpers::references::values::filtering_of; +use crate::planner_resolution::_helpers::references::values::method; +use crate::planner_resolution::_helpers::references::values::string; +use crate::planner_resolution::models::CursorFiltering; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; @@ -257,7 +258,7 @@ pub(crate) fn apply_deferred_locations<'py>( Ok((models, seeds)) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(resolve_ref_references, module)?)?; module.add_function(wrap_pyfunction!(resolve_dbt_ref_references, module)?)?; module.add_function(wrap_pyfunction!(resolve_udf_references, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs index 10bf24d5ed..14f27651ad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs @@ -1,10 +1,11 @@ //! Source contract validation and adapter-controlled projection rendering. -use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; -use crate::bindings::_helpers::planner_resolution::values::{ - cursor_wrap, filtering_of, method, string, -}; -use crate::bindings::_helpers::planner_strategy::values::input_error; +use crate::planner_resolution::_helpers::references::values::cursor_wrap; +use crate::planner_resolution::_helpers::references::values::filtering_of; +use crate::planner_resolution::_helpers::references::values::method; +use crate::planner_resolution::_helpers::references::values::string; +use crate::planner_resolution::models::CursorFiltering; +use crate::planner_strategy::main::values::input_error::input_error; use pyo3::IntoPyObject; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -352,7 +353,7 @@ pub(crate) fn resolve_sources( Ok(output) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(resolve_source_references, module)?)?; module.add_function(wrap_pyfunction!(render_source_read_relation, module)?)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs index d2d5e16d63..689cb59b7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs @@ -1,6 +1,6 @@ //! Boundary calls shared by reference and source resolution. -use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; +use crate::planner_resolution::models::CursorFiltering; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyString; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -11,11 +11,11 @@ use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to use sqlbuild_sqltext::resolution::main::cursor_subquery::cursor_subquery; use sqlbuild_sqltext::resolution::models::CursorFilter; -pub(super) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { +pub(crate) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { PyString::new(py, text).into_any() } -pub(super) fn method<'py>( +pub(crate) fn method<'py>( object: &Bound<'py, PyAny>, name: &str, values: &[(&str, Bound<'py, PyAny>)], @@ -23,7 +23,7 @@ pub(super) fn method<'py>( call_with(&native_field(object, name)?, values) } -pub(super) fn cursor_wrap( +pub(crate) fn cursor_wrap( adapter: &Bound<'_, PyAny>, filtering: &CursorFiltering<'_>, relation: String, @@ -93,7 +93,7 @@ fn cursor_operand( } /// The cursor filtering of a keyword-argument resolution request. -pub(super) fn filtering_of<'py>( +pub(crate) fn filtering_of<'py>( arguments: &KeywordArguments<'py>, ) -> PyResult> { Ok(CursorFiltering { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/future_safety.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/future_safety.rs index f2dbdb5769..7ef4b356af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/future_safety.rs @@ -1,8 +1,11 @@ //! Future cursor safety evidence and exception boundary. -use crate::bindings::_helpers::planner_resolution::cursor_safety::{ - horizon, replace, safety_duration, safety_error, timestamp_bound, utc_clock, -}; +use crate::planner_resolution::_helpers::cursor::cursor_safety::horizon; +use crate::planner_resolution::_helpers::cursor::cursor_safety::replace; +use crate::planner_resolution::_helpers::cursor::cursor_safety::safety_duration; +use crate::planner_resolution::_helpers::cursor::cursor_safety::safety_error; +use crate::planner_resolution::_helpers::cursor::cursor_safety::timestamp_bound; +use crate::planner_resolution::_helpers::cursor::cursor_safety::utc_clock; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; @@ -164,6 +167,6 @@ pub(crate) fn resolve_future_cursor_safety<'py>( replace(&bounds, &[("end", applied_end), ("future_safety", safety)]) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(resolve_future_cursor_safety, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs index faa31a2276..3c4f20b382 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs @@ -1,13 +1,16 @@ //! Maximum automatic-start recovery and evidence boundary. -use crate::bindings::_helpers::planner_resolution::cursor_bounds::apply_typed_cursor_replay_policy; -use crate::bindings::_helpers::planner_resolution::cursor_bounds::{ - optional_scalar, required_scalar, scalars, -}; -use crate::bindings::_helpers::planner_resolution::cursor_safety::{ - horizon, replace, safety_duration, safety_error, timestamp_bound, utc_clock, -}; -use crate::bindings::_helpers::planner_resolution::values::string; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::apply_typed_cursor_replay_policy; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::optional_scalar; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::required_scalar; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::scalars; +use crate::planner_resolution::_helpers::cursor::cursor_safety::horizon; +use crate::planner_resolution::_helpers::cursor::cursor_safety::replace; +use crate::planner_resolution::_helpers::cursor::cursor_safety::safety_duration; +use crate::planner_resolution::_helpers::cursor::cursor_safety::safety_error; +use crate::planner_resolution::_helpers::cursor::cursor_safety::timestamp_bound; +use crate::planner_resolution::_helpers::cursor::cursor_safety::utc_clock; +use crate::planner_resolution::_helpers::references::values::string; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -285,7 +288,7 @@ fn exceeded_message(evidence: &Bound<'_, PyAny>, non_idempotent: bool) -> PyResu )) } -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(maximum_allowed_start, module)?)?; module.add_function(wrap_pyfunction!(apply_maximum_start_policy, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/mod.rs new file mode 100644 index 0000000000..f07ca613f3 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod future_safety; +pub(crate) mod maximum_safety; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/constants.rs b/crates/sqlbuild-python-planner/src/planner_resolution/constants.rs new file mode 100644 index 0000000000..35c1634370 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/constants.rs @@ -0,0 +1,3 @@ +//! Public constants of planner resolution. + +pub(crate) const CURSOR_START: &str = "cursor_start"; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/config_str.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/config_str.rs new file mode 100644 index 0000000000..862fa084a4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/config_str.rs @@ -0,0 +1,12 @@ +//! One string-valued model configuration entry. + +use crate::planner_resolution::_helpers::cursor::cursor_ranges; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// One string-valued model configuration entry. +#[inline] +pub fn config_str<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { + cursor_ranges::config_str(model, key) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/cursor_safety_configs.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/cursor_safety_configs.rs new file mode 100644 index 0000000000..8bac8ea563 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/cursor_safety_configs.rs @@ -0,0 +1,15 @@ +//! The future-end and automatic-start policies effective for one model. + +use crate::planner_resolution::_helpers::cursor::cursor_ranges; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The future-end and automatic-start policies effective for one model. +#[inline] +pub fn cursor_safety_configs<'py>( + model: &Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + cursor_ranges::cursor_safety_configs(model, context) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/empty_cursor_inputs.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/empty_cursor_inputs.rs new file mode 100644 index 0000000000..93ab8f5f15 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/empty_cursor_inputs.rs @@ -0,0 +1,16 @@ +//! Empty inputs that leave a planner-owned window empty, and whether others have rows. + +use crate::planner_resolution::_helpers::cursor::cursor_ranges; +use crate::planner_resolution::models::CursorWindowInputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Empty inputs that leave a planner-owned window empty, and whether others have rows. +#[inline] +pub fn empty_cursor_inputs<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, +) -> PyResult<(Bound<'py, PyAny>, bool)> { + cursor_ranges::empty_cursor_inputs(model, inputs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/microbatch_range.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/microbatch_range.rs new file mode 100644 index 0000000000..b370ffbb28 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/microbatch_range.rs @@ -0,0 +1,16 @@ +//! Compute the real overall cursor range for microbatch batch splitting. + +use crate::planner_resolution::_helpers::cursor::cursor_ranges; +use crate::planner_resolution::models::CursorWindowInputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Compute the real overall cursor range for microbatch batch splitting. +#[inline] +pub fn microbatch_range<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, +) -> PyResult> { + cursor_ranges::microbatch_range(model, inputs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/mod.rs new file mode 100644 index 0000000000..221a4d8047 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/mod.rs @@ -0,0 +1,16 @@ +pub mod config_str; +pub mod cursor_safety_configs; +pub mod empty_cursor_inputs; +pub mod microbatch_range; +pub mod model_cursor_overrides; +pub mod normalize_cursor_snapshot_grain; +pub mod plan_cursor_bounds; +pub mod register_cursor_bounds; +pub mod register_cursor_overrides; +pub mod register_cursor_ranges; +pub mod register_cursor_warnings; +pub mod register_cursor_windows; +pub mod resolve_effective_timestamp_grain; +pub mod resolve_start_cursor_config; +pub mod type_cursor_overrides; +pub mod with_cursor_safety_warnings; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/model_cursor_overrides.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/model_cursor_overrides.rs new file mode 100644 index 0000000000..fa58db9799 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/model_cursor_overrides.rs @@ -0,0 +1,16 @@ +//! Resolve typed cursor overrides to generic start/end for one model. + +use crate::planner_resolution::_helpers::cursor::cursor_ranges; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Resolve typed cursor overrides to generic start/end for one model. +#[inline] +pub fn model_cursor_overrides<'py>( + model: &Bound<'py, PyAny>, + cursor_overrides: &Bound<'py, PyAny>, + (start, end): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + cursor_ranges::model_cursor_overrides(model, cursor_overrides, (start, end)) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/normalize_cursor_snapshot_grain.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/normalize_cursor_snapshot_grain.rs new file mode 100644 index 0000000000..0562231cb4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/normalize_cursor_snapshot_grain.rs @@ -0,0 +1,16 @@ +//! Normalize cursor snapshot grain. + +use crate::planner_resolution::_helpers::cursor::cursor_windows; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Normalize cursor snapshot grain. +#[inline] +pub fn normalize_cursor_snapshot_grain<'py>( + cursor_snapshot: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, + effective_grain: &Bound<'py, PyAny>, +) -> PyResult> { + cursor_windows::normalize_cursor_snapshot_grain(cursor_snapshot, cursor_type, effective_grain) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/plan_cursor_bounds.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/plan_cursor_bounds.rs new file mode 100644 index 0000000000..193df91965 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/plan_cursor_bounds.rs @@ -0,0 +1,16 @@ +//! Compute cursor bounds for inclusion on the plan entry. + +use crate::planner_resolution::_helpers::cursor::cursor_ranges; +use crate::planner_resolution::models::CursorWindowInputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Compute cursor bounds for inclusion on the plan entry. +#[inline] +pub fn plan_cursor_bounds<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, +) -> PyResult> { + cursor_ranges::plan_cursor_bounds(model, inputs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_bounds.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_bounds.rs new file mode 100644 index 0000000000..830c594d63 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_bounds.rs @@ -0,0 +1,12 @@ +//! Register the cursor bounds functions and classes. + +use crate::planner_resolution::_helpers::cursor::cursor_bounds; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the cursor bounds functions and classes. +#[inline] +pub fn register_cursor_bounds(module: &Bound<'_, PyModule>) -> PyResult<()> { + cursor_bounds::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_overrides.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_overrides.rs new file mode 100644 index 0000000000..5b1d19dab6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_overrides.rs @@ -0,0 +1,12 @@ +//! Register the cursor overrides functions and classes. + +use crate::planner_resolution::_helpers::cursor::cursor_overrides; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the cursor overrides functions and classes. +#[inline] +pub fn register_cursor_overrides(module: &Bound<'_, PyModule>) -> PyResult<()> { + cursor_overrides::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs new file mode 100644 index 0000000000..85c893558d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs @@ -0,0 +1,12 @@ +//! Register the cursor ranges functions and classes. + +use crate::planner_resolution::_helpers::cursor::cursor_ranges; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the cursor ranges functions and classes. +#[inline] +pub fn register_cursor_ranges(module: &Bound<'_, PyModule>) -> PyResult<()> { + cursor_ranges::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_warnings.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_warnings.rs new file mode 100644 index 0000000000..f0c033ae7d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_warnings.rs @@ -0,0 +1,12 @@ +//! Register the cursor warnings functions and classes. + +use crate::planner_resolution::_helpers::cursor::cursor_warnings; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the cursor warnings functions and classes. +#[inline] +pub fn register_cursor_warnings(module: &Bound<'_, PyModule>) -> PyResult<()> { + cursor_warnings::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_windows.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_windows.rs new file mode 100644 index 0000000000..ddda2d8f12 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_windows.rs @@ -0,0 +1,12 @@ +//! Register the cursor windows functions and classes. + +use crate::planner_resolution::_helpers::cursor::cursor_windows; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the cursor windows functions and classes. +#[inline] +pub fn register_cursor_windows(module: &Bound<'_, PyModule>) -> PyResult<()> { + cursor_windows::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_effective_timestamp_grain.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_effective_timestamp_grain.rs new file mode 100644 index 0000000000..6e492dc47e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_effective_timestamp_grain.rs @@ -0,0 +1,22 @@ +//! Resolve effective timestamp grain. + +use crate::planner_resolution::_helpers::cursor::cursor_windows; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Resolve effective timestamp grain. +#[inline] +pub fn resolve_effective_timestamp_grain<'py>( + cursor_type: &Bound<'py, PyAny>, + downstream_grain: &Bound<'py, PyAny>, + cursor_input_grains: &Bound<'py, PyAny>, + microbatch_strategy: Option<&str>, +) -> PyResult> { + cursor_windows::resolve_effective_timestamp_grain( + cursor_type, + downstream_grain, + cursor_input_grains, + microbatch_strategy, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_start_cursor_config.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_start_cursor_config.rs new file mode 100644 index 0000000000..60e0697ee1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_start_cursor_config.rs @@ -0,0 +1,15 @@ +//! Resolve automatic-start policy with model config taking precedence. + +use crate::planner_resolution::_helpers::cursor::cursor_policies; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Resolve automatic-start policy with model config taking precedence. +#[inline] +pub fn resolve_start_cursor_config<'py>( + model: &Bound<'py, PyAny>, + project_config: &Bound<'py, PyAny>, +) -> PyResult> { + cursor_policies::resolve_start_cursor_config(model, project_config) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/type_cursor_overrides.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/type_cursor_overrides.rs new file mode 100644 index 0000000000..f49f4b313b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/type_cursor_overrides.rs @@ -0,0 +1,15 @@ +//! Untyped cursor overrides typed by the cursor type of the selected incremental models. + +use crate::planner_resolution::_helpers::cursor::cursor_overrides; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Untyped cursor overrides typed by the cursor type of the selected incremental models. +#[inline] +pub fn type_cursor_overrides<'py>( + overrides: &Bound<'py, PyAny>, + selected: &[Bound<'py, PyAny>], +) -> PyResult> { + cursor_overrides::type_cursor_overrides(overrides, selected) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/with_cursor_safety_warnings.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/with_cursor_safety_warnings.rs new file mode 100644 index 0000000000..ac4e3f080d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/with_cursor_safety_warnings.rs @@ -0,0 +1,16 @@ +//! `warnings` followed by the future-end and automatic-start warnings `bounds` carries. + +use crate::planner_resolution::_helpers::cursor::cursor_warnings; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `warnings` followed by the future-end and automatic-start warnings `bounds` carries. +#[inline] +pub fn with_cursor_safety_warnings<'py>( + warnings: &Bound<'py, PyAny>, + model_name: &Bound<'py, PyAny>, + bounds: &Bound<'py, PyAny>, +) -> PyResult> { + cursor_warnings::with_cursor_safety_warnings(warnings, model_name, bounds) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/mod.rs new file mode 100644 index 0000000000..be6ead50c4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/mod.rs @@ -0,0 +1,3 @@ +pub mod cursor; +pub mod references; +pub mod safety; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/apply_deferred_locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/apply_deferred_locations.rs new file mode 100644 index 0000000000..28529111fa --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/apply_deferred_locations.rs @@ -0,0 +1,23 @@ +//! Apply deferred locations. + +use crate::planner_resolution::_helpers::references::refs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Apply deferred locations. +#[inline] +pub fn apply_deferred_locations<'py>( + model_locations: &Bound<'py, PyDict>, + seed_locations: &Bound<'py, PyDict>, + deferred_locations: &Bound<'py, PyDict>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { + refs::apply_deferred_locations( + model_locations, + seed_locations, + deferred_locations, + selected_keys, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_function_locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_function_locations.rs new file mode 100644 index 0000000000..583e56dda5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_function_locations.rs @@ -0,0 +1,15 @@ +//! Build function locations. + +use crate::planner_resolution::_helpers::references::locations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Build function locations. +#[inline] +pub fn build_function_locations<'py>( + functions: &Bound<'py, PyAny>, +) -> PyResult> { + locations::build_function_locations(functions) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_model_locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_model_locations.rs new file mode 100644 index 0000000000..674abc8aea --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_model_locations.rs @@ -0,0 +1,13 @@ +//! Build model locations. + +use crate::planner_resolution::_helpers::references::locations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Build model locations. +#[inline] +pub fn build_model_locations<'py>(models: &Bound<'py, PyAny>) -> PyResult> { + locations::build_model_locations(models) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_seed_locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_seed_locations.rs new file mode 100644 index 0000000000..9df886fb51 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_seed_locations.rs @@ -0,0 +1,13 @@ +//! Build seed locations. + +use crate::planner_resolution::_helpers::references::locations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Build seed locations. +#[inline] +pub fn build_seed_locations<'py>(seeds: &Bound<'py, PyAny>) -> PyResult> { + locations::build_seed_locations(seeds) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/mod.rs new file mode 100644 index 0000000000..ebc871f35e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/mod.rs @@ -0,0 +1,11 @@ +pub mod apply_deferred_locations; +pub mod build_function_locations; +pub mod build_model_locations; +pub mod build_seed_locations; +pub mod model_hook_entries; +pub mod register_locations; +pub mod register_orchestration; +pub mod register_refs; +pub mod register_sources; +pub mod resolve_function_sql; +pub mod resolve_model_sql; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/model_hook_entries.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/model_hook_entries.rs new file mode 100644 index 0000000000..906c263d9a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/model_hook_entries.rs @@ -0,0 +1,17 @@ +//! One model's pre and post hooks with their SQL resolved. + +use crate::planner_resolution::_helpers::references::orchestration; +use crate::planner_resolution::models::Resolution; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// One model's pre and post hooks with their SQL resolved. +#[inline] +pub fn model_hook_entries<'py>( + model: &Bound<'py, PyAny>, + resolution: &Resolution<'py>, +) -> PyResult> { + orchestration::model_hook_entries(model, resolution) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs new file mode 100644 index 0000000000..ec44ce6a56 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs @@ -0,0 +1,12 @@ +//! Register the locations functions and classes. + +use crate::planner_resolution::_helpers::references::locations; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the locations functions and classes. +#[inline] +pub fn register_locations(module: &Bound<'_, PyModule>) -> PyResult<()> { + locations::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs new file mode 100644 index 0000000000..194040359c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs @@ -0,0 +1,12 @@ +//! Register the orchestration functions and classes. + +use crate::planner_resolution::_helpers::references::orchestration; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the orchestration functions and classes. +#[inline] +pub fn register_orchestration(module: &Bound<'_, PyModule>) -> PyResult<()> { + orchestration::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs new file mode 100644 index 0000000000..368e042aa8 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs @@ -0,0 +1,12 @@ +//! Register the refs functions and classes. + +use crate::planner_resolution::_helpers::references::refs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the refs functions and classes. +#[inline] +pub fn register_refs(module: &Bound<'_, PyModule>) -> PyResult<()> { + refs::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_sources.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_sources.rs new file mode 100644 index 0000000000..37f8b61408 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_sources.rs @@ -0,0 +1,12 @@ +//! Register the sources functions and classes. + +use crate::planner_resolution::_helpers::references::sources; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the sources functions and classes. +#[inline] +pub fn register_sources(module: &Bound<'_, PyModule>) -> PyResult<()> { + sources::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_function_sql.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_function_sql.rs new file mode 100644 index 0000000000..df5e846fae --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_function_sql.rs @@ -0,0 +1,17 @@ +//! Resolve function sql. + +use crate::planner_resolution::_helpers::references::orchestration; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// Resolve function sql. +#[inline] +pub fn resolve_function_sql<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + orchestration::resolve_function_sql(py, kwargs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_model_sql.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_model_sql.rs new file mode 100644 index 0000000000..cefc089062 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_model_sql.rs @@ -0,0 +1,18 @@ +//! One model's planned SQL: references resolved, cursor inputs filtered and intrinsics bound. + +use crate::planner_resolution::_helpers::references::orchestration; +use crate::planner_resolution::models::ModelSqlRequest; +use crate::planner_resolution::models::Resolution; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// One model's planned SQL: references resolved, cursor inputs filtered and intrinsics bound. +#[inline] +pub fn resolve_model_sql<'py>( + resolution: &Resolution<'py>, + context: &Bound<'py, PyAny>, + request: &ModelSqlRequest<'py>, +) -> PyResult { + orchestration::resolve_model_sql(resolution, context, request) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/maximum_allowed_start.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/maximum_allowed_start.rs new file mode 100644 index 0000000000..acae9f4346 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/maximum_allowed_start.rs @@ -0,0 +1,16 @@ +//! Maximum allowed start. + +use crate::planner_resolution::_helpers::safety::maximum_safety; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; + +/// Maximum allowed start. +#[inline] +pub fn maximum_allowed_start<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult { + maximum_safety::maximum_allowed_start(py, kwargs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/mod.rs new file mode 100644 index 0000000000..bac12ee9a3 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/mod.rs @@ -0,0 +1,3 @@ +pub mod maximum_allowed_start; +pub mod register_future_safety; +pub mod register_maximum_safety; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_future_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_future_safety.rs new file mode 100644 index 0000000000..64dafdc018 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_future_safety.rs @@ -0,0 +1,12 @@ +//! Register the future safety functions and classes. + +use crate::planner_resolution::_helpers::safety::future_safety; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the future safety functions and classes. +#[inline] +pub fn register_future_safety(module: &Bound<'_, PyModule>) -> PyResult<()> { + future_safety::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_maximum_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_maximum_safety.rs new file mode 100644 index 0000000000..f912b1d4a6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_maximum_safety.rs @@ -0,0 +1,12 @@ +//! Register the maximum safety functions and classes. + +use crate::planner_resolution::_helpers::safety::maximum_safety; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the maximum safety functions and classes. +#[inline] +pub fn register_maximum_safety(module: &Bound<'_, PyModule>) -> PyResult<()> { + maximum_safety::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/mod.rs new file mode 100644 index 0000000000..e8dd7a3eda --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/mod.rs @@ -0,0 +1,6 @@ +//! Planner reference resolution with Python adapter rendering callbacks. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/models.rs b/crates/sqlbuild-python-planner/src/planner_resolution/models.rs new file mode 100644 index 0000000000..17f70498bc --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/models.rs @@ -0,0 +1,168 @@ +//! Public records of planner resolution. + +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_phases::main::inputs::class_field::class_field; +use crate::planner_resolution::_helpers::references::refs::resolve_dbt_ref_references; +use crate::planner_resolution::_helpers::references::refs::resolve_refs; +use crate::planner_resolution::_helpers::references::refs::resolve_table_function_references; +use crate::planner_resolution::_helpers::references::refs::resolve_udf_references; +use crate::planner_resolution::_helpers::references::sources::resolve_sources; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyList; +use pyo3::types::PyString; +use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::attachments::main::assert_no_unresolved_sql_markers::assert_no_unresolved_sql_markers; + +/// The planner inputs that decide one model's plan-time cursor window. +#[derive(Debug)] +pub struct CursorWindowInputs<'py> { + pub snapshot: Bound<'py, PyAny>, + pub backfill: Bound<'py, PyAny>, + pub overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub runtime_owned: bool, + pub policies: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub invocation_time: Bound<'py, PyAny>, + pub full_refresh: bool, + pub replaces_relation: bool, + pub cursor_input_relations: Bound<'py, PyAny>, +} + +/// The cursor window that filters the cursor inputs one resolution reads. +#[derive(Debug)] +pub(crate) struct CursorFiltering<'py> { + pub bounds: Bound<'py, PyAny>, + pub filter_inputs: Bound<'py, PyAny>, + pub cursor_type: Bound<'py, PyAny>, + pub lower_bound_inclusive: bool, +} + +/// One model's SQL resolution decisions. +#[derive(Debug)] +pub struct ModelSqlRequest<'py> { + pub model: Bound<'py, PyAny>, + pub snapshot: Bound<'py, PyAny>, + pub backfill: Bound<'py, PyAny>, + pub full_refresh: bool, + pub cursor_overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub replaces_relation: bool, + pub full_rebuild_cause: Option, +} + +/// What resolving one model's, hook's or function's references reads. +#[derive(Debug)] +pub struct Resolution<'py> { + pub adapter: Bound<'py, PyAny>, + pub(crate) external: Bound<'py, PyAny>, + pub(crate) source_map: Bound<'py, PyAny>, + pub(crate) source_warehouse_columns: Bound<'py, PyAny>, + pub(crate) model_locations: Bound<'py, PyAny>, + pub(crate) seed_locations: Bound<'py, PyAny>, + pub(crate) function_locations: Bound<'py, PyAny>, +} + +impl<'py> Resolution<'py> { + /// Resolution against a `ModelPlanContext`'s relations. + pub fn of_context( + adapter: Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, + external: Bound<'py, PyAny>, + ) -> PyResult { + Ok(Self { + adapter, + external, + source_map: class_field(context, |c: &ModelPlanContext| &c.source_map)?, + source_warehouse_columns: class_field(context, |c: &ModelPlanContext| { + &c.source_warehouse_columns + })?, + model_locations: class_field(context, |c: &ModelPlanContext| &c.model_locations)?, + seed_locations: class_field(context, |c: &ModelPlanContext| &c.seed_locations)?, + function_locations: class_field(context, |c: &ModelPlanContext| &c.function_locations)?, + }) + } + + pub(crate) fn resolve(&self, sql: &str, filtering: &CursorFiltering<'py>) -> PyResult { + let py = self.adapter.py(); + let resolved: String = resolve_sources( + sql, + (&self.source_map, &self.source_warehouse_columns), + &self.adapter, + filtering, + )?; + let resolved: String = resolve_refs( + &resolved, + (&self.model_locations, &self.seed_locations), + &self.adapter, + filtering, + )?; + let resolved: String = resolve_dbt_ref_references(&resolved, Some(&self.external))?; + let locations = if self.function_locations.is_truthy()? { + self.function_locations.clone() + } else { + PyDict::new(py).into_any() + }; + let resolved: String = resolve_udf_references(&resolved, &locations, &self.adapter)?; + resolve_table_function_references(&resolved, &locations, &self.adapter) + } + + pub(crate) fn assert_resolved(&self, sql: &str, label: &str) -> PyResult<()> { + let py = self.adapter.py(); + assert_no_unresolved_sql_markers( + py, + &PyString::new(py, sql), + &format!("{label} planned SQL"), + ) + } + + pub(crate) fn unbounded(&self, sql: Bound<'py, PyAny>, label: &str) -> PyResult { + let py = self.adapter.py(); + let resolved = self.resolve( + &sql.extract::()?, + &CursorFiltering { + bounds: py.None().into_bound(py), + filter_inputs: PyDict::new(py).into_any(), + cursor_type: py.None().into_bound(py), + lower_bound_inclusive: true, + }, + )?; + self.assert_resolved(&resolved, label)?; + Ok(resolved) + } + + pub(crate) fn hook( + &self, + value: Bound<'py, PyAny>, + label: &str, + ) -> PyResult> { + let py = value.py(); + if value.is_instance(&record_class(py, RecordId::SqlHookEntry)?)? { + let kwargs = PyDict::new(py); + kwargs.set_item( + "statement", + self.unbounded(native_field(&value, "statement")?, label)?, + )?; + return replaced_keywords(&value, &kwargs); + } + let is_list = value.is_instance_of::(); + if is_list || value.is_instance_of::() { + let mut resolved: Vec> = Vec::new(); + for (index, item) in value.try_iter()?.enumerate() { + resolved.push(self.hook(item?, &format!("{label}[{index}]"))?); + } + return if is_list { + Ok(PyList::new(py, resolved)?.into_any()) + } else { + Ok(PyTuple::new(py, resolved)?.into_any()) + }; + } + Ok(value) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs similarity index 80% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs index d941f2657e..3cb657c7f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs @@ -1,10 +1,10 @@ //! Scenario artifact dataclasses and adapter identifier callbacks. -use crate::bindings::_helpers::planner_scenario::constants::{ - SCENARIO_PLAN_HASH_COLLISION, SCENARIO_PLAN_INVALID_HASH_PREFIX, - SCENARIO_PLAN_RELATION_COLLISION, -}; -use crate::bindings::_helpers::planner_scenario::values::{error, string}; +use crate::planner_scenario::_helpers::values::error; +use crate::planner_scenario::_helpers::values::string; +use crate::planner_scenario::constants::SCENARIO_PLAN_HASH_COLLISION; +use crate::planner_scenario::constants::SCENARIO_PLAN_INVALID_HASH_PREFIX; +use crate::planner_scenario::constants::SCENARIO_PLAN_RELATION_COLLISION; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_scenario::main::{hash_prefix, parse_artifact_name}; @@ -266,3 +266,45 @@ pub(crate) fn build_scenario_relation_map<'py>( ], ) } + +/// `build_scenario_artifact_name` as a function of `module`. +pub(crate) fn build_scenario_artifact_name_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_scenario_artifact_name, module) +} + +/// `build_scenario_hash_index` as a function of `module`. +pub(crate) fn build_scenario_hash_index_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_scenario_hash_index, module) +} + +/// `build_scenario_relation_map` as a function of `module`. +pub(crate) fn build_scenario_relation_map_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_scenario_relation_map, module) +} + +/// `compute_scenario_hash_prefix` as a function of `module`. +pub(crate) fn compute_scenario_hash_prefix_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(compute_scenario_hash_prefix, module) +} + +/// `is_scenario_artifact_physical_name` as a function of `module`. +pub(crate) fn scenario_artifact_physical_name_check_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(is_scenario_artifact_physical_name, module) +} + +/// `parse_scenario_artifact_physical_name` as a function of `module`. +pub(crate) fn parse_scenario_artifact_physical_name_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(parse_scenario_artifact_physical_name, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs index ab2e5b1c87..93f9447935 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs @@ -1,13 +1,13 @@ //! Assemble CLI scenario plans while retaining adapter and executor boundaries. -use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_hash_index; -use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_relation_map; -use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; -use crate::bindings::_helpers::planner_scenario::execution::build_scenario_execution_plan; -use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; -use crate::bindings::_helpers::planner_scenario::models::ScenarioRelations; -use crate::bindings::_helpers::planner_scenario::relations::build_scenario_relation_plan; -use crate::bindings::_helpers::planner_scenario::values::string; +use crate::planner_scenario::_helpers::artifacts::build_scenario_hash_index; +use crate::planner_scenario::_helpers::artifacts::build_scenario_relation_map; +use crate::planner_scenario::_helpers::execution::build_scenario_execution_plan; +use crate::planner_scenario::_helpers::graph::plan_scenario_graph; +use crate::planner_scenario::_helpers::relations::build_scenario_relation_plan; +use crate::planner_scenario::_helpers::values::string; +use crate::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; +use crate::planner_scenario::models::ScenarioRelations; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -190,3 +190,10 @@ fn raise_for_warnings(warnings: &Bound<'_, PyAny>) -> PyResult<()> { } Ok(()) } + +/// `build_cli_scenario_plan` as a function of `module`. +pub(crate) fn build_cli_scenario_plan_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_cli_scenario_plan, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs index b26466437c..44e9e48e1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs @@ -1,12 +1,12 @@ //! Existing seed and function plan dataclass construction. -use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; -use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; -use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_UNKNOWN_SEED; -use crate::bindings::_helpers::planner_scenario::{ - models::SqlContext, sql::required_target, values::error, -}; -use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; +use crate::planner_identity::main::build_compiled_function_fingerprint_sql::build_compiled_function_fingerprint_sql; +use crate::planner_resolution::main::references::resolve_function_sql::resolve_function_sql; +use crate::planner_scenario::_helpers::sql::required_target; +use crate::planner_scenario::_helpers::values::error; +use crate::planner_scenario::constants::SCENARIO_PLAN_UNKNOWN_SEED; +use crate::planner_scenario::models::SqlContext; +use crate::planner_strategy::main::model_entries::extract_seed_columns::extract_seed_columns; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyString; use pyo3::types::{PyDictMethods, PyTuple}; @@ -19,7 +19,7 @@ use sqlbuild_python_project::resource_views::main::seed_reads::key::key; use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; -pub(super) fn seed_entries<'py>( +pub(crate) fn seed_entries<'py>( project: &Bound<'py, PyAny>, graph: &Bound<'py, PyAny>, relations: &Bound<'py, PyAny>, @@ -74,7 +74,7 @@ pub(super) fn seed_entries<'py>( PyTuple::new(py, result) } -pub(super) fn function_entry<'py>( +pub(crate) fn function_entry<'py>( function: &Bound<'py, PyAny>, context: &SqlContext<'py>, locations: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs index db3da0dc18..6b97bb5ce5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs @@ -1,21 +1,28 @@ //! Assemble scenario execution plans through existing planner and adapter APIs. -use crate::bindings::_helpers::planner_entries::model_entries::plan_model; -use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; -use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; -use crate::bindings::_helpers::planner_scenario::constants::{ - CURSOR_INTRINSIC_NAMES, CURSOR_TYPE_KEY, INCREMENTAL_MODE_KEY, INTEGER_CURSOR_TYPE, - MICROBATCH_INCREMENTAL_MODE, SCENARIO_EXPECTED_COLUMNS_HELP, SCENARIO_PLAN_EXPECTED_COLUMNS, - SCENARIO_PLAN_INVALID_FIXTURE, -}; -use crate::bindings::_helpers::planner_scenario::helper_scope::scenario_helper_scope; -use crate::bindings::_helpers::planner_scenario::{ - entries::{function_entry, seed_entries}, - fixtures::{extract_fixtures, fixture_plans}, - models::SqlContext, - sql::{required_target, resolve_model_dbt_refs, resolve_sql, source_label}, - values::{error, string}, -}; +use crate::planner_entries::main::entries::plan_model::plan_model; +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_resolution::main::references::build_function_locations::build_function_locations; +use crate::planner_scenario::_helpers::entries::function_entry; +use crate::planner_scenario::_helpers::entries::seed_entries; +use crate::planner_scenario::_helpers::fixtures::extract_fixtures; +use crate::planner_scenario::_helpers::fixtures::fixture_plans; +use crate::planner_scenario::_helpers::helper_scope::scenario_helper_scope; +use crate::planner_scenario::_helpers::sql::required_target; +use crate::planner_scenario::_helpers::sql::resolve_model_dbt_refs; +use crate::planner_scenario::_helpers::sql::resolve_sql; +use crate::planner_scenario::_helpers::sql::source_label; +use crate::planner_scenario::_helpers::values::error; +use crate::planner_scenario::_helpers::values::string; +use crate::planner_scenario::constants::CURSOR_INTRINSIC_NAMES; +use crate::planner_scenario::constants::CURSOR_TYPE_KEY; +use crate::planner_scenario::constants::INCREMENTAL_MODE_KEY; +use crate::planner_scenario::constants::INTEGER_CURSOR_TYPE; +use crate::planner_scenario::constants::MICROBATCH_INCREMENTAL_MODE; +use crate::planner_scenario::constants::SCENARIO_EXPECTED_COLUMNS_HELP; +use crate::planner_scenario::constants::SCENARIO_PLAN_EXPECTED_COLUMNS; +use crate::planner_scenario::constants::SCENARIO_PLAN_INVALID_FIXTURE; +use crate::planner_scenario::models::SqlContext; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::compiler::main::scenario_expected_column_message::scenario_expected_column_message; @@ -440,3 +447,10 @@ fn scenario_cursor_window<'py>(model: &Bound<'py, PyAny>) -> PyResult( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_scenario_execution_plan, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/fixtures.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/fixtures.rs index 669fa0088e..4cd3040cdd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/fixtures.rs @@ -1,12 +1,13 @@ //! Materialize fixture SQL plans while preserving adapter and completion boundaries. -use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_MISSING_FIXTURE_SQL; -use crate::bindings::_helpers::planner_scenario::helper_scope::scenario_helper_scope; -use crate::bindings::_helpers::planner_scenario::{ - models::SqlContext, - sql::{required_target, resolve_sql, source_label}, - values::{error, string}, -}; +use crate::planner_scenario::_helpers::helper_scope::scenario_helper_scope; +use crate::planner_scenario::_helpers::sql::required_target; +use crate::planner_scenario::_helpers::sql::resolve_sql; +use crate::planner_scenario::_helpers::sql::source_label; +use crate::planner_scenario::_helpers::values::error; +use crate::planner_scenario::_helpers::values::string; +use crate::planner_scenario::constants::SCENARIO_PLAN_MISSING_FIXTURE_SQL; +use crate::planner_scenario::models::SqlContext; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::compiler::main::scenario_reader_ctes::scenario_reader_ctes; @@ -24,7 +25,7 @@ use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use std::collections::{BTreeMap, BTreeSet}; -pub(super) fn extract_fixtures<'py>( +pub(crate) fn extract_fixtures<'py>( scenario: &Bound<'py, PyAny>, prefix: &str, ) -> PyResult> { @@ -34,7 +35,7 @@ pub(super) fn extract_fixtures<'py>( ) } -pub(super) fn fixture_plans<'py>( +pub(crate) fn fixture_plans<'py>( scenario: &Bound<'py, PyAny>, graph: &Bound<'py, PyAny>, context: &SqlContext<'py>, @@ -77,7 +78,7 @@ pub(super) fn fixture_plans<'py>( )?); }; let mut sql = authored.clone(); - if kind != super::constants::DBT_REF && !overrides.is_none() { + if kind != crate::planner_scenario::constants::DBT_REF && !overrides.is_none() { sql = mapping_get_or(overrides, (resource, &name), sql)?; } let sql = resolve_sql( @@ -190,3 +191,10 @@ pub(crate) fn build_scenario_fixture_plans<'py>( &args.value("fixture_sql_overrides")?, ) } + +/// `build_scenario_fixture_plans` as a function of `module`. +pub(crate) fn build_scenario_fixture_plans_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_scenario_fixture_plans, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/graph.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/graph.rs index e369c1a279..36ac0ad16d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/graph.rs @@ -1,7 +1,7 @@ //! Convert project views into pure scenario dependency facts. -use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; -use crate::bindings::_helpers::planner_scenario::values::string; +use crate::planner_scenario::_helpers::values::string; +use crate::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; use pyo3::types::PyTuple; use sqlbuild_analysis::planner_scenario::main::graph::graph; @@ -116,3 +116,10 @@ pub(crate) fn plan_scenario_graph<'py>( } Ok((plan, PyTuple::new(py, warnings)?)) } + +/// `plan_scenario_graph` as a function of `module`. +pub(crate) fn plan_scenario_graph_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(plan_scenario_graph, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/helper_scope.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/helper_scope.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/helper_scope.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/helper_scope.rs index 3b66e666b8..42a7e3491f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/helper_scope.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/helper_scope.rs @@ -4,8 +4,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::compiler::main::scenario_helper_scope::scenario_helper_scope as helper_scope; use sqlbuild_analysis::compiler::models::{ScenarioHelperScope, ScenarioScopeInput}; -use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_INTERNAL; -use crate::bindings::_helpers::planner_scenario::{models::SqlContext, values::error}; +use crate::planner_scenario::_helpers::values::error; +use crate::planner_scenario::constants::SCENARIO_PLAN_INTERNAL; +use crate::planner_scenario::models::SqlContext; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; @@ -18,7 +19,7 @@ const FIXTURE_LOCATIONS: [(&str, &str); 4] = [ ]; /// The scenario's helper scope, where a check reading a fixture by name reads its relation. -pub(super) fn scenario_helper_scope( +pub(crate) fn scenario_helper_scope( scenario: &Bound<'_, PyAny>, context: &SqlContext<'_>, ) -> PyResult { diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/mod.rs new file mode 100644 index 0000000000..9721320a6b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod artifacts; +pub(crate) mod cli; +pub(crate) mod entries; +pub(crate) mod execution; +pub(crate) mod fixtures; +pub(crate) mod graph; +pub(crate) mod helper_scope; +pub(crate) mod relations; +pub(crate) mod sql; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/relations.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/relations.rs index 02f6055a68..7d29b4f4d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/relations.rs @@ -1,11 +1,11 @@ //! Build scenario relation locations through adapter qualification callbacks. -use crate::bindings::_helpers::planner_scenario::constants::{REF, SOURCE}; -use crate::bindings::_helpers::planner_scenario::constants::{ - SCENARIO_PLAN_INTERNAL, SCENARIO_PLAN_MISSING_RELATION_TARGET, -}; -use crate::bindings::_helpers::planner_scenario::models::ScenarioRelations; -use crate::bindings::_helpers::planner_scenario::values::error; +use crate::planner_scenario::_helpers::values::error; +use crate::planner_scenario::constants::REF; +use crate::planner_scenario::constants::SCENARIO_PLAN_INTERNAL; +use crate::planner_scenario::constants::SCENARIO_PLAN_MISSING_RELATION_TARGET; +use crate::planner_scenario::constants::SOURCE; +use crate::planner_scenario::models::ScenarioRelations; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -184,3 +184,10 @@ fn target_for_artifact<'py>( ], ) } + +/// `build_scenario_relation_plan_of` as a function of `module`. +pub(crate) fn build_scenario_relation_plan_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_scenario_relation_plan_of, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs index 323632ffd2..85fc59560f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs @@ -1,12 +1,10 @@ //! Resolve scenario SQL through the shared native scanner and adapter callbacks. -use crate::bindings::_helpers::planner_scenario::constants::{ - SCENARIO_PLAN_MISSING_RELATION_TARGET, SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, -}; -use crate::bindings::_helpers::planner_scenario::{ - models::SqlContext, - values::{error, string}, -}; +use crate::planner_scenario::_helpers::values::error; +use crate::planner_scenario::_helpers::values::string; +use crate::planner_scenario::constants::SCENARIO_PLAN_MISSING_RELATION_TARGET; +use crate::planner_scenario::constants::SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER; +use crate::planner_scenario::models::SqlContext; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyStringMethods}; use sqlbuild_analysis::planner_scenario::{ @@ -20,7 +18,7 @@ use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; use sqlbuild_sqltext::sql_scan::models::Unclosed; -pub(super) fn resolve_sql( +pub(crate) fn resolve_sql( context: &SqlContext<'_>, sql: &str, project_sources: bool, @@ -41,13 +39,13 @@ pub(super) fn resolve_sql( ), (3, "dbt_ref_fixture_locations"), ] { - if project_sources && kind != super::constants::SOURCE_MARKER { + if project_sources && kind != crate::planner_scenario::constants::SOURCE_MARKER { continue; } let locations = context.relations.getattr(field)?; result = replace_markers(&result, kind, Some(syntax.clone().into()), |name, marker| { let target = mapping_get(&locations, name)?; - let target = if target.is_none() {target} else if kind == super::constants::SOURCE_MARKER { + let target = if target.is_none() {target} else if kind == crate::planner_scenario::constants::SOURCE_MARKER { object(py, render_source_relation(&target, Some(&context.adapter.clone()).filter(|value| !value.is_none()))?)? } else {native_field(&target, "qualified_name")?}; if target.is_none() { @@ -68,7 +66,7 @@ pub(super) fn resolve_sql( Ok(result) } -pub(super) fn resolve_model_dbt_refs(sql: &str, relations: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn resolve_model_dbt_refs(sql: &str, relations: &Bound<'_, PyAny>) -> PyResult { let locations = native_field(relations, "dbt_ref_fixture_locations")?; replace_markers(sql, 3, None, |name, marker| { let target = mapping_get(&locations, name)?; @@ -91,7 +89,7 @@ pub(super) fn resolve_model_dbt_refs(sql: &str, relations: &Bound<'_, PyAny>) -> }) } -pub(super) fn source_label(scenario: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn source_label(scenario: &Bound<'_, PyAny>) -> PyResult { let source = native_field(scenario, "source_path")?; if source.is_truthy()? { Ok(source.str()?.to_str()?.to_owned()) @@ -100,7 +98,7 @@ pub(super) fn source_label(scenario: &Bound<'_, PyAny>) -> PyResult { } } -pub(super) fn required_target<'py>( +pub(crate) fn required_target<'py>( relations: &Bound<'py, PyAny>, field: &str, name: &str, @@ -147,3 +145,10 @@ pub(crate) fn resolve_scenario_check_sql( }; resolve_sql(&context, &args.value("sql")?.extract::()?, false) } + +/// `resolve_scenario_check_sql` as a function of `module`. +pub(crate) fn resolve_scenario_check_sql_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(resolve_scenario_check_sql, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/values.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/values.rs index b15067e5d4..9cbae543c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/values.rs @@ -6,11 +6,11 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; -pub(super) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { +pub(crate) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { PyString::new(py, text).into_any() } -pub(super) fn error( +pub(crate) fn error( py: Python<'_>, message: &str, code: &str, diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/constants.rs b/crates/sqlbuild-python-planner/src/planner_scenario/constants.rs new file mode 100644 index 0000000000..8d76b2f40c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/constants.rs @@ -0,0 +1,44 @@ +//! Public constants of planner scenario. + +pub(crate) const REF: &str = "ref"; + +pub(crate) const SOURCE: &str = "source"; + +pub(crate) const SOURCE_MARKER: usize = 2; + +pub(crate) const DBT_REF: &str = "dbt_ref"; + +pub(crate) const SCENARIO_PLAN_INVALID_HASH_PREFIX: &str = "S501"; + +pub(crate) const SCENARIO_PLAN_HASH_COLLISION: &str = "S502"; + +pub(crate) const SCENARIO_PLAN_RELATION_COLLISION: &str = "S503"; + +pub(crate) const SCENARIO_PLAN_GRAPH_VALIDATION: &str = "S504"; + +pub(crate) const SCENARIO_PLAN_UNKNOWN_SEED: &str = "S507"; + +pub(crate) const SCENARIO_PLAN_MISSING_FIXTURE_SQL: &str = "S508"; + +pub(crate) const SCENARIO_PLAN_MISSING_RELATION_TARGET: &str = "S509"; + +pub(crate) const SCENARIO_PLAN_INVALID_FIXTURE: &str = "S510"; + +pub(crate) const SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER: &str = "S511"; + +pub(crate) const SCENARIO_PLAN_EXPECTED_COLUMNS: &str = "S512"; + +pub(crate) const SCENARIO_EXPECTED_COLUMNS_HELP: &str = "list every column the model outputs in the expected CTE, and remove columns it does not output"; + +pub(crate) const SCENARIO_PLAN_INTERNAL: &str = "S599"; + +/// The zero-argument cursor intrinsics a model query can call. +pub(crate) const CURSOR_INTRINSIC_NAMES: [&str; 2] = ["__cursor_start", "__cursor_end"]; + +pub(crate) const INCREMENTAL_MODE_KEY: &str = "incremental_mode"; + +pub(crate) const MICROBATCH_INCREMENTAL_MODE: &str = "microbatch"; + +pub(crate) const CURSOR_TYPE_KEY: &str = "cursor_type"; + +pub(crate) const INTEGER_CURSOR_TYPE: &str = "integer"; diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_cli_scenario_plan_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_cli_scenario_plan_function.rs new file mode 100644 index 0000000000..7d35e0fb95 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_cli_scenario_plan_function.rs @@ -0,0 +1,11 @@ +//! `build_cli_scenario_plan` as a function of `module`. + +use crate::planner_scenario::_helpers::cli; + +/// `build_cli_scenario_plan` as a function of `module`. +#[inline] +pub fn build_cli_scenario_plan_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + cli::build_cli_scenario_plan_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_artifact_name_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_artifact_name_function.rs new file mode 100644 index 0000000000..bddf06c54b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_artifact_name_function.rs @@ -0,0 +1,11 @@ +//! `build_scenario_artifact_name` as a function of `module`. + +use crate::planner_scenario::_helpers::artifacts; + +/// `build_scenario_artifact_name` as a function of `module`. +#[inline] +pub fn build_scenario_artifact_name_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + artifacts::build_scenario_artifact_name_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_execution_plan_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_execution_plan_function.rs new file mode 100644 index 0000000000..19ac88867d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_execution_plan_function.rs @@ -0,0 +1,11 @@ +//! `build_scenario_execution_plan` as a function of `module`. + +use crate::planner_scenario::_helpers::execution; + +/// `build_scenario_execution_plan` as a function of `module`. +#[inline] +pub fn build_scenario_execution_plan_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + execution::build_scenario_execution_plan_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_fixture_plans_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_fixture_plans_function.rs new file mode 100644 index 0000000000..ebda2bde0c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_fixture_plans_function.rs @@ -0,0 +1,11 @@ +//! `build_scenario_fixture_plans` as a function of `module`. + +use crate::planner_scenario::_helpers::fixtures; + +/// `build_scenario_fixture_plans` as a function of `module`. +#[inline] +pub fn build_scenario_fixture_plans_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + fixtures::build_scenario_fixture_plans_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_hash_index_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_hash_index_function.rs new file mode 100644 index 0000000000..f2fd71baae --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_hash_index_function.rs @@ -0,0 +1,11 @@ +//! `build_scenario_hash_index` as a function of `module`. + +use crate::planner_scenario::_helpers::artifacts; + +/// `build_scenario_hash_index` as a function of `module`. +#[inline] +pub fn build_scenario_hash_index_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + artifacts::build_scenario_hash_index_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_map_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_map_function.rs new file mode 100644 index 0000000000..efa0cc0e7d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_map_function.rs @@ -0,0 +1,11 @@ +//! `build_scenario_relation_map` as a function of `module`. + +use crate::planner_scenario::_helpers::artifacts; + +/// `build_scenario_relation_map` as a function of `module`. +#[inline] +pub fn build_scenario_relation_map_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + artifacts::build_scenario_relation_map_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_plan_of_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_plan_of_function.rs new file mode 100644 index 0000000000..a1e335da76 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_plan_of_function.rs @@ -0,0 +1,11 @@ +//! `build_scenario_relation_plan_of` as a function of `module`. + +use crate::planner_scenario::_helpers::relations; + +/// `build_scenario_relation_plan_of` as a function of `module`. +#[inline] +pub fn build_scenario_relation_plan_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + relations::build_scenario_relation_plan_of_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/compute_scenario_hash_prefix_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/compute_scenario_hash_prefix_function.rs new file mode 100644 index 0000000000..c27bc1c187 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/compute_scenario_hash_prefix_function.rs @@ -0,0 +1,11 @@ +//! `compute_scenario_hash_prefix` as a function of `module`. + +use crate::planner_scenario::_helpers::artifacts; + +/// `compute_scenario_hash_prefix` as a function of `module`. +#[inline] +pub fn compute_scenario_hash_prefix_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + artifacts::compute_scenario_hash_prefix_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/is_scenario_artifact_physical_name.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/is_scenario_artifact_physical_name.rs new file mode 100644 index 0000000000..c683c80ad9 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/is_scenario_artifact_physical_name.rs @@ -0,0 +1,9 @@ +//! Is scenario artifact physical name. + +use crate::planner_scenario::_helpers::artifacts; + +/// Is scenario artifact physical name. +#[inline] +pub fn is_scenario_artifact_physical_name(name: &str) -> bool { + artifacts::is_scenario_artifact_physical_name(name) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/mod.rs new file mode 100644 index 0000000000..646c568a99 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/mod.rs @@ -0,0 +1,14 @@ +pub mod build_cli_scenario_plan_function; +pub mod build_scenario_artifact_name_function; +pub mod build_scenario_execution_plan_function; +pub mod build_scenario_fixture_plans_function; +pub mod build_scenario_hash_index_function; +pub mod build_scenario_relation_map_function; +pub mod build_scenario_relation_plan_of_function; +pub mod compute_scenario_hash_prefix_function; +pub mod is_scenario_artifact_physical_name; +pub mod parse_scenario_artifact_physical_name_function; +pub mod plan_scenario_graph; +pub mod plan_scenario_graph_function; +pub mod resolve_scenario_check_sql_function; +pub mod scenario_artifact_physical_name_check_function; diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/parse_scenario_artifact_physical_name_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/parse_scenario_artifact_physical_name_function.rs new file mode 100644 index 0000000000..0267a2a3a8 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/parse_scenario_artifact_physical_name_function.rs @@ -0,0 +1,11 @@ +//! `parse_scenario_artifact_physical_name` as a function of `module`. + +use crate::planner_scenario::_helpers::artifacts; + +/// `parse_scenario_artifact_physical_name` as a function of `module`. +#[inline] +pub fn parse_scenario_artifact_physical_name_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + artifacts::parse_scenario_artifact_physical_name_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph.rs new file mode 100644 index 0000000000..c2fcd07eda --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph.rs @@ -0,0 +1,23 @@ +//! Plan scenario graph. + +use crate::planner_scenario::_helpers::graph; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Plan scenario graph. +#[inline] +pub fn plan_scenario_graph<'py>( + scenario: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, + sql_analysis_enabled: bool, + sql_analysis_dialect: Option<&str>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyTuple>)> { + graph::plan_scenario_graph( + scenario, + project, + sql_analysis_enabled, + sql_analysis_dialect, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph_function.rs new file mode 100644 index 0000000000..40d1bfbecb --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph_function.rs @@ -0,0 +1,11 @@ +//! `plan_scenario_graph` as a function of `module`. + +use crate::planner_scenario::_helpers::graph; + +/// `plan_scenario_graph` as a function of `module`. +#[inline] +pub fn plan_scenario_graph_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + graph::plan_scenario_graph_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/resolve_scenario_check_sql_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/resolve_scenario_check_sql_function.rs new file mode 100644 index 0000000000..f3301813d6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/resolve_scenario_check_sql_function.rs @@ -0,0 +1,11 @@ +//! `resolve_scenario_check_sql` as a function of `module`. + +use crate::planner_scenario::_helpers::sql; + +/// `resolve_scenario_check_sql` as a function of `module`. +#[inline] +pub fn resolve_scenario_check_sql_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + sql::resolve_scenario_check_sql_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/scenario_artifact_physical_name_check_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/scenario_artifact_physical_name_check_function.rs new file mode 100644 index 0000000000..9b75fff57c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/scenario_artifact_physical_name_check_function.rs @@ -0,0 +1,11 @@ +//! `is_scenario_artifact_physical_name` as a function of `module`. + +use crate::planner_scenario::_helpers::artifacts; + +/// `is_scenario_artifact_physical_name` as a function of `module`. +#[inline] +pub fn scenario_artifact_physical_name_check_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + artifacts::scenario_artifact_physical_name_check_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/mod.rs b/crates/sqlbuild-python-planner/src/planner_scenario/mod.rs new file mode 100644 index 0000000000..86c8fdf294 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/mod.rs @@ -0,0 +1,6 @@ +//! Python boundaries for native scenario planning. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/models.rs b/crates/sqlbuild-python-planner/src/planner_scenario/models.rs new file mode 100644 index 0000000000..d9e42cc382 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/models.rs @@ -0,0 +1,23 @@ +//! Public records of planner scenario. + +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; + +#[derive(Debug)] +pub(crate) struct SqlContext<'py> { + pub adapter: Bound<'py, PyAny>, + pub relations: Bound<'py, PyAny>, + pub syntax: Bound<'py, PyAny>, + pub label: String, +} + +/// One scenario's graph and the relation map and qualification that locate its relations. +#[derive(Debug)] +pub(crate) struct ScenarioRelations<'py> { + pub project: Bound<'py, PyAny>, + pub graph_plan: Bound<'py, PyAny>, + pub relation_map: Bound<'py, PyAny>, + pub render_qualified_name: Bound<'py, PyAny>, + pub database: Bound<'py, PyAny>, + pub schema: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs index 6617268137..a2bd8bcdd4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs @@ -5,9 +5,9 @@ use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::models::GraphIndexes; use std::collections::HashSet; -use crate::bindings::_helpers::planner_strategy::audits::{ - attachment, outside_target_lineage, resolve_effective_run_scope, -}; +use crate::planner_strategy::_helpers::audits::attachment; +use crate::planner_strategy::_helpers::audits::outside_target_lineage; +use crate::planner_strategy::_helpers::audits::resolve_effective_run_scope; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; @@ -286,3 +286,10 @@ pub(crate) fn build_selected_audit_entries<'py>( } Ok(entries) } + +/// `plan_audit_of` as a function of `module`. +pub(crate) fn plan_audit_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(plan_audit_of, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs index 3c4e27c50a..af0f930f91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs @@ -15,7 +15,9 @@ use sqlbuild_analysis::planner_strategy::main::effective_run_scope::effective_ru use sqlbuild_analysis::planner_strategy::main::singular_attachment::singular_attachment; use sqlbuild_analysis::planner_strategy::models::{AuditAttachmentKind, AuditRunScope}; -use crate::bindings::_helpers::planner_strategy::values::{input_error, optional_string, text}; +use crate::planner_strategy::_helpers::values::input_error; +use crate::planner_strategy::_helpers::values::optional_string; +use crate::planner_strategy::_helpers::values::text; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_inputs.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_inputs.rs index 37b06bfb0a..7945307cc4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_inputs.rs @@ -7,9 +7,10 @@ use pyo3::types::{PyBool, PyDict, PyFrozenSet, PyString, PyTuple}; use sqlbuild_analysis::planner_strategy::main::cursor_input_column::cursor_input_column_failure; use sqlbuild_analysis::planner_strategy::models::{CursorInputColumnCheck, CursorInputColumnOwner}; -use crate::bindings::_helpers::planner_identity::metadata::or_empty; -use crate::bindings::_helpers::planner_strategy::lineage::resolve_lineage_reference; -use crate::bindings::_helpers::planner_strategy::values::{input_error, text}; +use crate::planner_identity::main::or_empty::or_empty; +use crate::planner_strategy::_helpers::lineage::resolve_lineage_reference; +use crate::planner_strategy::_helpers::values::input_error; +use crate::planner_strategy::_helpers::values::text; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -358,3 +359,24 @@ pub(crate) fn runtime_cursor_producer_names<'py>( } PyFrozenSet::new(py, names) } + +/// `cursor_input_columns_of` as a function of `module`. +pub(crate) fn cursor_input_columns_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(cursor_input_columns_of, module) +} + +/// `cursor_input_relations_of` as a function of `module`. +pub(crate) fn cursor_input_relations_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(cursor_input_relations_of, module) +} + +/// `runtime_cursor_producer_names` as a function of `module`. +pub(crate) fn runtime_cursor_producer_names_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(runtime_cursor_producer_names, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs index 126310f583..986b260bfd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs @@ -10,8 +10,10 @@ use sqlbuild_analysis::planner_strategy::main::cursor_type_warning::cursor_type_ use sqlbuild_analysis::planner_strategy::models::{CursorType, CursorTypeCheck}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::planner_changes::results::planner_enum; -use crate::bindings::_helpers::planner_strategy::values::{optional_string, plan_warning, text}; +use crate::planner_changes::main::planner_enum::planner_enum; +use crate::planner_strategy::_helpers::values::optional_string; +use crate::planner_strategy::_helpers::values::plan_warning; +use crate::planner_strategy::_helpers::values::text; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs index 40fdaa4647..4126c3d166 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs @@ -9,10 +9,10 @@ use sqlbuild_analysis::planner_strategy::main::lineage_reference::lineage_refere use sqlbuild_analysis::planner_strategy::models::{ProducerFacts, ReferenceIdentity}; use sqlbuild_analysis::planner_strategy::types::ReferenceGraph; -use crate::bindings::_helpers::planner_strategy::microbatches::{ - CAP_ACTIONS, microbatch_limit_config, -}; -use crate::bindings::_helpers::planner_strategy::values::{input_error, text}; +use crate::planner_strategy::_helpers::microbatches::CAP_ACTIONS; +use crate::planner_strategy::_helpers::microbatches::microbatch_limit_config; +use crate::planner_strategy::_helpers::values::input_error; +use crate::planner_strategy::_helpers::values::text; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs index 9153633367..06cab7ab70 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs @@ -1,6 +1,6 @@ //! Microbatch limits applied to planned model entries in CLI, project and model precedence. -use crate::bindings::_helpers::planner_changes::results::native_enum; +use crate::planner_changes::main::native_enum::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction}; @@ -14,11 +14,13 @@ use sqlbuild_core::plan_display::errors::UnknownCursorGrain; use sqlbuild_core::plan_display::main::effective_timestamp_grain::effective_timestamp_grain; use sqlbuild_core::plan_display::models::TimestampGrainFacts; -use crate::bindings::_helpers::plan_output::display::resolve_effective_microbatch_batch_size; -use crate::bindings::_helpers::planner_strategy::microbatches::{ - CAP_ACTIONS, capped_microbatch_bounds, count_microbatches, -}; -use crate::bindings::_helpers::planner_strategy::values::{input_error, optional_string, text}; +use crate::plan_output::main::resolve_effective_microbatch_batch_size::resolve_effective_microbatch_batch_size; +use crate::planner_strategy::_helpers::microbatches::CAP_ACTIONS; +use crate::planner_strategy::_helpers::microbatches::capped_microbatch_bounds; +use crate::planner_strategy::_helpers::microbatches::count_microbatches; +use crate::planner_strategy::_helpers::values::input_error; +use crate::planner_strategy::_helpers::values::optional_string; +use crate::planner_strategy::_helpers::values::text; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; @@ -257,3 +259,17 @@ pub(crate) fn apply_configured_microbatch_limits<'py>( let model_limit: Bound<'py, PyAny> = native_field(&entry, "microbatch_limit")?; with_warnings(limited(&entry, model_limit, declared)?, warnings) } + +/// `apply_configured_microbatch_limits` as a function of `module`. +pub(crate) fn apply_configured_microbatch_limits_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(apply_configured_microbatch_limits, module) +} + +/// `apply_microbatch_limit` as a function of `module`. +pub(crate) fn apply_microbatch_limit_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(apply_microbatch_limit, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatches.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatches.rs index 239ab716a3..19457fb1ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatches.rs @@ -18,8 +18,8 @@ use sqlbuild_analysis::planner_strategy::types::Timeline; use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; use sqlbuild_cursor::algebra::models::{DurationDigits, DurationSpan}; -use crate::bindings::_helpers::plan_output::display::aligned_timestamp_bounds; -use crate::bindings::_helpers::planner_strategy::values::text; +use crate::plan_output::main::aligned_timestamp_bounds::aligned_timestamp_bounds; +use crate::planner_strategy::_helpers::values::text; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/mod.rs new file mode 100644 index 0000000000..e1e98c41e6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod audit_entries; +pub(crate) mod audits; +pub(crate) mod cursor_inputs; +pub(crate) mod cursor_types; +pub(crate) mod lineage; +pub(crate) mod microbatch_limits; +pub(crate) mod microbatches; +pub(crate) mod model_entries; +pub(crate) mod strategy; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs index 6d08220ae0..ef946bd1da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs @@ -17,17 +17,18 @@ use sqlbuild_model_config::execution_limits::main::parse_execution_limit::parse_ use sqlbuild_model_config::execution_limits::models::ExecutionLimit; use sqlbuild_model_config::model_validation::constants::PHYSICAL_STORAGE_KEYS; -use crate::bindings::_helpers::planner_strategy::cursor_types::{ - cursor_type_warning_record, warehouse_column_types, -}; -use crate::bindings::_helpers::planner_strategy::microbatch_limits::{replay_grain, unknown_grain}; -use crate::bindings::_helpers::planner_strategy::microbatches::microbatch_limit_config; -use crate::bindings::_helpers::planner_strategy::strategy::{ - model_warning_records, resolve_schema_actions, warning_inputs, -}; -use crate::bindings::_helpers::planner_strategy::values::{ - input_error, optional_string, plan_warning, text, -}; +use crate::planner_strategy::_helpers::cursor_types::cursor_type_warning_record; +use crate::planner_strategy::_helpers::cursor_types::warehouse_column_types; +use crate::planner_strategy::_helpers::microbatch_limits::replay_grain; +use crate::planner_strategy::_helpers::microbatch_limits::unknown_grain; +use crate::planner_strategy::_helpers::microbatches::microbatch_limit_config; +use crate::planner_strategy::_helpers::strategy::model_warning_records; +use crate::planner_strategy::_helpers::strategy::resolve_schema_actions; +use crate::planner_strategy::_helpers::strategy::warning_inputs; +use crate::planner_strategy::_helpers::values::input_error; +use crate::planner_strategy::_helpers::values::optional_string; +use crate::planner_strategy::_helpers::values::plan_warning; +use crate::planner_strategy::_helpers::values::text; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; @@ -783,3 +784,10 @@ pub(crate) fn assemble_model_plan_entry<'py>( let plan_entry: Bound<'py, PyAny> = new_record(py, RecordId::ModelPlanEntry, (), Some(&entry))?; Ok((plan_entry, warnings)) } + +/// `resolve_microbatch_lookback` as a function of `module`. +pub(crate) fn resolve_microbatch_lookback_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(resolve_microbatch_lookback, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs index d68f6cd890..96aebe881f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs @@ -1,6 +1,6 @@ //! Per-model plan actions, schema actions and plan warnings. -use crate::bindings::_helpers::planner_changes::results::native_enum; +use crate::planner_changes::main::native_enum::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; @@ -14,11 +14,16 @@ use sqlbuild_analysis::planner_strategy::models::{ ModelStrategy, PlanAction, PlanReason, SchemaActionKind, WarningInputs, }; -use crate::bindings::_helpers::planner_changes::results::planner_enum; -use crate::bindings::_helpers::planner_strategy::values::{ - change_facts, config_value, finding_rows, input_error, materialization_value, - model_materialization, optional_string, plan_warning, text, -}; +use crate::planner_changes::main::planner_enum::planner_enum; +use crate::planner_strategy::_helpers::values::change_facts; +use crate::planner_strategy::_helpers::values::config_value; +use crate::planner_strategy::_helpers::values::finding_rows; +use crate::planner_strategy::_helpers::values::input_error; +use crate::planner_strategy::_helpers::values::materialization_value; +use crate::planner_strategy::_helpers::values::model_materialization; +use crate::planner_strategy::_helpers::values::optional_string; +use crate::planner_strategy::_helpers::values::plan_warning; +use crate::planner_strategy::_helpers::values::text; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs index 22e3989765..4f7a256c1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs @@ -1,6 +1,6 @@ //! Conversions between the planner's Python models and the native strategy rows. -use crate::bindings::_helpers::planner_changes::results::native_enum; +use crate::planner_changes::main::native_enum::native_enum; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString}; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs new file mode 100644 index 0000000000..9135742077 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs @@ -0,0 +1,17 @@ +//! Select attached audits by their logical target and singular audits by dependency overlap. + +use crate::planner_strategy::_helpers::audit_entries; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use std::collections::HashSet; + +/// Select attached audits by their logical target and singular audits by dependency overlap. +#[inline] +pub fn audit_is_selected( + audit: &Bound<'_, PyAny>, + selected_keys: &HashSet, +) -> PyResult { + audit_entries::audit_is_selected(audit, selected_keys) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs new file mode 100644 index 0000000000..13a48a73a1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs @@ -0,0 +1,15 @@ +//! Assemble selected audits in compiled-project order without querying lineage for an empty selection. + +use crate::planner_strategy::_helpers::audit_entries; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyList; + +/// Assemble selected audits in compiled-project order without querying lineage for an empty selection. +#[inline] +pub fn build_selected_audit_entries<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + audit_entries::build_selected_audit_entries(request) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/mod.rs new file mode 100644 index 0000000000..31bd8b5c4c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/mod.rs @@ -0,0 +1,3 @@ +pub mod audit_is_selected; +pub mod build_selected_audit_entries; +pub mod plan_audit_of_function; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/plan_audit_of_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/plan_audit_of_function.rs new file mode 100644 index 0000000000..7900703a0c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/plan_audit_of_function.rs @@ -0,0 +1,11 @@ +//! `plan_audit_of` as a function of `module`. + +use crate::planner_strategy::_helpers::audit_entries; + +/// `plan_audit_of` as a function of `module`. +#[inline] +pub fn plan_audit_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + audit_entries::plan_audit_of_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/mod.rs new file mode 100644 index 0000000000..00e5e61db2 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/mod.rs @@ -0,0 +1 @@ +pub mod register_audits; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs new file mode 100644 index 0000000000..9c28e5c8d6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs @@ -0,0 +1,12 @@ +//! Register the audits functions and classes. + +use crate::planner_strategy::_helpers::audits; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the audits functions and classes. +#[inline] +pub fn register_audits(module: &Bound<'_, PyModule>) -> PyResult<()> { + audits::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/build_cursor_input_relations.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/build_cursor_input_relations.rs new file mode 100644 index 0000000000..a64b2148fa --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/build_cursor_input_relations.rs @@ -0,0 +1,18 @@ +//! Cursor-bearing input relations of an incremental model, for runtime range discovery. + +use crate::planner_strategy::_helpers::cursor_inputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; +use sqlbuild_python_project::planner_strategy::models::CursorInputLookups; + +/// Cursor-bearing input relations of an incremental model, for runtime range discovery. +#[inline] +pub fn build_cursor_input_relations<'py>( + (model, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + cursor_column: &Bound<'py, PyAny>, + lookups: &CursorInputLookups<'py>, +) -> PyResult> { + cursor_inputs::build_cursor_input_relations((model, adapter), cursor_column, lookups) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_columns_of_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_columns_of_function.rs new file mode 100644 index 0000000000..0d30b3e9f6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_columns_of_function.rs @@ -0,0 +1,11 @@ +//! `cursor_input_columns_of` as a function of `module`. + +use crate::planner_strategy::_helpers::cursor_inputs; + +/// `cursor_input_columns_of` as a function of `module`. +#[inline] +pub fn cursor_input_columns_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + cursor_inputs::cursor_input_columns_of_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_relations_of_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_relations_of_function.rs new file mode 100644 index 0000000000..482ac222e8 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_relations_of_function.rs @@ -0,0 +1,11 @@ +//! `cursor_input_relations_of` as a function of `module`. + +use crate::planner_strategy::_helpers::cursor_inputs; + +/// `cursor_input_relations_of` as a function of `module`. +#[inline] +pub fn cursor_input_relations_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + cursor_inputs::cursor_input_relations_of_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/mod.rs new file mode 100644 index 0000000000..1106186cea --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/mod.rs @@ -0,0 +1,6 @@ +pub mod build_cursor_input_relations; +pub mod cursor_input_columns_of_function; +pub mod cursor_input_relations_of_function; +pub mod reject_unexposed_cursor_input_columns; +pub mod runtime_cursor_producer_names; +pub mod runtime_cursor_producer_names_function; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/reject_unexposed_cursor_input_columns.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/reject_unexposed_cursor_input_columns.rs new file mode 100644 index 0000000000..4c82b8ebce --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/reject_unexposed_cursor_input_columns.rs @@ -0,0 +1,17 @@ +//! Reject authored cursor input columns a contract or the source warehouse metadata does not expose. + +use crate::planner_strategy::_helpers::cursor_inputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_python_project::planner_strategy::models::CursorInputLookups; + +/// Reject authored cursor input columns a contract or the source warehouse metadata does not expose. +#[inline] +pub fn reject_unexposed_cursor_input_columns<'py>( + model: &Bound<'py, PyAny>, + cursor_column: &Bound<'py, PyAny>, + lookups: &CursorInputLookups<'py>, +) -> PyResult<()> { + cursor_inputs::reject_unexposed_cursor_input_columns(model, cursor_column, lookups) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names.rs new file mode 100644 index 0000000000..0f5bdd1309 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names.rs @@ -0,0 +1,16 @@ +//! Selected models and seeds, and selected sources with loaders, whose relations this invocation produces. + +use crate::planner_strategy::_helpers::cursor_inputs; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyFrozenSet; + +/// Selected models and seeds, and selected sources with loaders, whose relations this invocation produces. +#[inline] +pub fn runtime_cursor_producer_names<'py>( + scope: &Bound<'py, PyAny>, + source_map: &Bound<'py, PyAny>, +) -> PyResult> { + cursor_inputs::runtime_cursor_producer_names(scope, source_map) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names_function.rs new file mode 100644 index 0000000000..720442a149 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names_function.rs @@ -0,0 +1,11 @@ +//! `runtime_cursor_producer_names` as a function of `module`. + +use crate::planner_strategy::_helpers::cursor_inputs; + +/// `runtime_cursor_producer_names` as a function of `module`. +#[inline] +pub fn runtime_cursor_producer_names_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + cursor_inputs::runtime_cursor_producer_names_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/classify_cursor_sql_type.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/classify_cursor_sql_type.rs new file mode 100644 index 0000000000..48408a7f55 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/classify_cursor_sql_type.rs @@ -0,0 +1,16 @@ +//! The `CursorType` a supported cross-adapter SQL type holds, or `None`. + +use crate::planner_strategy::_helpers::cursor_types; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// The `CursorType` a supported cross-adapter SQL type holds, or `None`. +#[inline] +pub fn classify_cursor_sql_type<'py>( + py: Python<'py>, + sql_type: &str, +) -> PyResult>> { + cursor_types::classify_cursor_sql_type(py, sql_type) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/mod.rs new file mode 100644 index 0000000000..67abd3e832 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/mod.rs @@ -0,0 +1,2 @@ +pub mod classify_cursor_sql_type; +pub mod register_cursor_types; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/register_cursor_types.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/register_cursor_types.rs new file mode 100644 index 0000000000..90b4e247a4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/register_cursor_types.rs @@ -0,0 +1,12 @@ +//! Register the cursor types functions and classes. + +use crate::planner_strategy::_helpers::cursor_types; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the cursor types functions and classes. +#[inline] +pub fn register_cursor_types(module: &Bound<'_, PyModule>) -> PyResult<()> { + cursor_types::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/mod.rs new file mode 100644 index 0000000000..b4626ea813 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/mod.rs @@ -0,0 +1,2 @@ +pub mod reject_capped_watermark_inputs; +pub mod resolve_lineage_reference; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/reject_capped_watermark_inputs.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/reject_capped_watermark_inputs.rs new file mode 100644 index 0000000000..bf260c1a71 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/reject_capped_watermark_inputs.rs @@ -0,0 +1,17 @@ +//! Reject capped producers feeding a microbatch model's watermark inputs. + +use crate::planner_strategy::_helpers::lineage; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_python_project::native_host::models::ModelLookup; + +/// Reject capped producers feeding a microbatch model's watermark inputs. +#[inline] +pub fn reject_capped_watermark_inputs<'py>( + model: &Bound<'py, PyAny>, + models_by_name: ModelLookup, + functions_by_name: Option>, +) -> PyResult<()> { + lineage::reject_capped_watermark_inputs(model, models_by_name, functions_by_name) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/resolve_lineage_reference.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/resolve_lineage_reference.rs new file mode 100644 index 0000000000..b1e42955b5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/resolve_lineage_reference.rs @@ -0,0 +1,18 @@ +//! The reference one named input resolves to through model and SQL function references. + +use crate::planner_strategy::_helpers::lineage; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_python_project::native_host::models::ModelLookup; + +/// The reference one named input resolves to through model and SQL function references. +#[inline] +pub fn resolve_lineage_reference<'py>( + model: &Bound<'py, PyAny>, + input_name: &str, + models_by_name: ModelLookup, + functions_by_name: Bound<'py, PyAny>, +) -> PyResult> { + lineage::resolve_lineage_reference(model, input_name, models_by_name, functions_by_name) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs new file mode 100644 index 0000000000..56eeaa8c66 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs @@ -0,0 +1,16 @@ +//! Apply CLI, project, and model batch limits in their documented precedence order. + +use crate::planner_strategy::_helpers::microbatch_limits; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Apply CLI, project, and model batch limits in their documented precedence order. +#[inline] +pub fn apply_configured_microbatch_limits<'py>( + entry: &Bound<'py, PyAny>, + inputs: &Bound<'py, PyAny>, +) -> PyResult> { + microbatch_limits::apply_configured_microbatch_limits(entry, inputs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits_function.rs new file mode 100644 index 0000000000..31f11af667 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits_function.rs @@ -0,0 +1,11 @@ +//! `apply_configured_microbatch_limits` as a function of `module`. + +use crate::planner_strategy::_helpers::microbatch_limits; + +/// `apply_configured_microbatch_limits` as a function of `module`. +#[inline] +pub fn apply_configured_microbatch_limits_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + microbatch_limits::apply_configured_microbatch_limits_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_microbatch_limit_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_microbatch_limit_function.rs new file mode 100644 index 0000000000..a5d0e2ad19 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_microbatch_limit_function.rs @@ -0,0 +1,11 @@ +//! `apply_microbatch_limit` as a function of `module`. + +use crate::planner_strategy::_helpers::microbatch_limits; + +/// `apply_microbatch_limit` as a function of `module`. +#[inline] +pub fn apply_microbatch_limit_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + microbatch_limits::apply_microbatch_limit_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/mod.rs new file mode 100644 index 0000000000..102c9624e7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/mod.rs @@ -0,0 +1,3 @@ +pub mod apply_configured_microbatch_limits; +pub mod apply_configured_microbatch_limits_function; +pub mod apply_microbatch_limit_function; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/mod.rs new file mode 100644 index 0000000000..de713ce3e5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/mod.rs @@ -0,0 +1 @@ +pub mod register_microbatches; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/register_microbatches.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/register_microbatches.rs new file mode 100644 index 0000000000..046d394df8 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/register_microbatches.rs @@ -0,0 +1,12 @@ +//! Register the microbatches functions and classes. + +use crate::planner_strategy::_helpers::microbatches; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the microbatches functions and classes. +#[inline] +pub fn register_microbatches(module: &Bound<'_, PyModule>) -> PyResult<()> { + microbatches::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/mod.rs new file mode 100644 index 0000000000..66ad816139 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/mod.rs @@ -0,0 +1,10 @@ +pub mod audit_entries; +pub mod audits; +pub mod cursor_inputs; +pub mod cursor_types; +pub mod lineage; +pub mod microbatch_limits; +pub mod microbatches; +pub mod model_entries; +pub mod strategy; +pub mod values; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/assemble_model_plan_entry.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/assemble_model_plan_entry.rs new file mode 100644 index 0000000000..a8e1cb9758 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/assemble_model_plan_entry.rs @@ -0,0 +1,16 @@ +//! Build a model plan entry and its warnings from resolved plan decisions. + +use crate::planner_strategy::_helpers::model_entries; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; +use sqlbuild_python_project::planner_strategy::models::ModelEntryDecisions; + +/// Build a model plan entry and its warnings from resolved plan decisions. +#[inline] +pub fn assemble_model_plan_entry<'py>( + request: &ModelEntryDecisions<'py>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyTuple>)> { + model_entries::assemble_model_plan_entry(request) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/extract_seed_columns.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/extract_seed_columns.rs new file mode 100644 index 0000000000..65e9d663b9 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/extract_seed_columns.rs @@ -0,0 +1,13 @@ +//! Declared seed columns with types, as `ColumnInfo`. + +use crate::planner_strategy::_helpers::model_entries; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// Declared seed columns with types, as `ColumnInfo`. +#[inline] +pub fn extract_seed_columns<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + model_entries::extract_seed_columns(seed) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/mod.rs new file mode 100644 index 0000000000..4725a86594 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/mod.rs @@ -0,0 +1,4 @@ +pub mod assemble_model_plan_entry; +pub mod extract_seed_columns; +pub mod resolve_microbatch_lookback; +pub mod resolve_microbatch_lookback_function; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback.rs new file mode 100644 index 0000000000..e3d9a3fb48 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback.rs @@ -0,0 +1,15 @@ +//! Resolve explicit or default lookback for one microbatch model. + +use crate::planner_strategy::_helpers::model_entries; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Resolve explicit or default lookback for one microbatch model. +#[inline] +pub fn resolve_microbatch_lookback( + model: &Bound<'_, PyAny>, + effective_grain: Option, +) -> PyResult> { + model_entries::resolve_microbatch_lookback(model, effective_grain) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback_function.rs new file mode 100644 index 0000000000..b3ec92fffb --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback_function.rs @@ -0,0 +1,11 @@ +//! `resolve_microbatch_lookback` as a function of `module`. + +use crate::planner_strategy::_helpers::model_entries; + +/// `resolve_microbatch_lookback` as a function of `module`. +#[inline] +pub fn resolve_microbatch_lookback_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + model_entries::resolve_microbatch_lookback_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/materialization_of.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/materialization_of.rs new file mode 100644 index 0000000000..0d82ec45b1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/materialization_of.rs @@ -0,0 +1,12 @@ +//! The `MaterializationType` of a model's `materialized` config: table when unset, custom when unknown. + +use crate::planner_strategy::_helpers::strategy; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The `MaterializationType` of a model's `materialized` config: table when unset, custom when unknown. +#[inline] +pub fn materialization_of<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + strategy::materialization_of(model) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/mod.rs new file mode 100644 index 0000000000..d6a3768b49 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/mod.rs @@ -0,0 +1,3 @@ +pub mod materialization_of; +pub mod register_strategy; +pub mod resolve_model_plan_action; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs new file mode 100644 index 0000000000..a19d038880 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs @@ -0,0 +1,12 @@ +//! Register the strategy functions and classes. + +use crate::planner_strategy::_helpers::strategy; +use pyo3::prelude::Bound; +use pyo3::prelude::PyModule; +use pyo3::prelude::PyResult; + +/// Register the strategy functions and classes. +#[inline] +pub fn register_strategy(module: &Bound<'_, PyModule>) -> PyResult<()> { + strategy::register(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/resolve_model_plan_action.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/resolve_model_plan_action.rs new file mode 100644 index 0000000000..1a05091e9d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/resolve_model_plan_action.rs @@ -0,0 +1,17 @@ +//! The plan action and reason for one model. + +use crate::planner_strategy::_helpers::strategy; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyTuple; + +/// The plan action and reason for one model. +#[inline] +pub fn resolve_model_plan_action<'py>( + model: &Bound<'py, PyAny>, + change_result: &Bound<'py, PyAny>, + full_refresh: bool, +) -> PyResult> { + strategy::resolve_model_plan_action(model, change_result, full_refresh) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs new file mode 100644 index 0000000000..4bca20251b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs @@ -0,0 +1,12 @@ +//! The native error `class(message, code=code)`. + +use crate::planner_strategy::_helpers::values; +use pyo3::prelude::PyErr; +use pyo3::prelude::Python; +use sqlbuild_python_base::records::types::ErrorId; + +/// The native error `class(message, code=code)`. +#[inline] +pub fn input_error(py: Python<'_>, class: ErrorId, message: &str, code: Option<&str>) -> PyErr { + values::input_error(py, class, message, code) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/values/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/mod.rs new file mode 100644 index 0000000000..a67b53e73f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/mod.rs @@ -0,0 +1,2 @@ +pub mod input_error; +pub mod text; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/values/text.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/text.rs new file mode 100644 index 0000000000..9e2cd8e8ca --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/text.rs @@ -0,0 +1,12 @@ +//! `str(value)` of a Python string or string enum. + +use crate::planner_strategy::_helpers::values; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// `str(value)` of a Python string or string enum. +#[inline] +pub fn text(value: &Bound<'_, PyAny>) -> PyResult { + values::text(value) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/mod.rs new file mode 100644 index 0000000000..acc20e2651 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/mod.rs @@ -0,0 +1,4 @@ +//! Planner strategy for model entries: audits, cursor inputs and types, lineage and microbatches. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs index 877eb665ee..912bb907c5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs @@ -1,18 +1,23 @@ //! Cursor MIN/MAX inspection of selected incremental models and their watermark inputs. -use crate::bindings::_helpers::planner_resolution::cursor_policies::resolve_start_cursor_config; -use crate::bindings::_helpers::planner_resolution::cursor_windows::normalize_cursor_snapshot_grain; -use crate::bindings::_helpers::planner_resolution::cursor_windows::resolve_effective_timestamp_grain; -use crate::bindings::_helpers::planner_resolution::maximum_safety::maximum_allowed_start; -use crate::bindings::_helpers::planner_strategy::lineage::resolve_lineage_reference; -use crate::bindings::_helpers::planner_warehouse::constants::{ - CURSOR_ALGEBRA_ERROR, CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, DATE_VALUE, LOG_DEBUG_EVENT, - PLANNER_LOGGER, QUERY_RECORD, RECORD_QUERY, RUN_BOUNDED, RUN_RECORDED_QUERY, -}; -use crate::bindings::_helpers::planner_warehouse::snapshot::{ - SnapshotRequest, inspection_concurrency, -}; -use crate::bindings::_helpers::planner_warehouse::watermarks::validate_watermark_column; +use crate::planner_resolution::main::cursor::normalize_cursor_snapshot_grain::normalize_cursor_snapshot_grain; +use crate::planner_resolution::main::cursor::resolve_effective_timestamp_grain::resolve_effective_timestamp_grain; +use crate::planner_resolution::main::cursor::resolve_start_cursor_config::resolve_start_cursor_config; +use crate::planner_resolution::main::safety::maximum_allowed_start::maximum_allowed_start; +use crate::planner_strategy::main::lineage::resolve_lineage_reference::resolve_lineage_reference; +use crate::planner_warehouse::_helpers::inspection::snapshot::inspection_concurrency; +use crate::planner_warehouse::_helpers::inspection::watermarks::validate_watermark_column; +use crate::planner_warehouse::constants::CURSOR_ALGEBRA_ERROR; +use crate::planner_warehouse::constants::CURSOR_TYPE_INTEGER; +use crate::planner_warehouse::constants::CURSOR_TYPE_TIMESTAMP; +use crate::planner_warehouse::constants::DATE_VALUE; +use crate::planner_warehouse::constants::LOG_DEBUG_EVENT; +use crate::planner_warehouse::constants::PLANNER_LOGGER; +use crate::planner_warehouse::constants::QUERY_RECORD; +use crate::planner_warehouse::constants::RECORD_QUERY; +use crate::planner_warehouse::constants::RUN_BOUNDED; +use crate::planner_warehouse::constants::RUN_RECORDED_QUERY; +use crate::planner_warehouse::models::SnapshotRequest; use pyo3::IntoPyObject; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs new file mode 100644 index 0000000000..f2f8383dbb --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs @@ -0,0 +1,112 @@ +//! Migration fingerprints computed at most once per model definition and reused across runs. + +use crate::planner_warehouse::models::FingerprintWork; +use pyo3::PyRefMut; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyFloat; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::imported::imported; +use std::hash::{DefaultHasher, Hash, Hasher}; + +pub(crate) const CACHE_DIRECTORY_NAME: &str = "migration-fingerprints-v1"; +pub(crate) const CACHE_DATABASE_NAME: &str = "migration-fingerprints.sqlite3"; +const SQLITE_TIMEOUT_SECONDS: f64 = 0.1; +pub(crate) const CREATE_CACHE_TABLE_SQL: &str = "\nCREATE TABLE IF NOT EXISTS migration_fingerprint (\n cache_key TEXT PRIMARY KEY,\n payload TEXT NOT NULL\n)\n"; +pub(crate) const INSERT_SQL: &str = + "INSERT OR REPLACE INTO migration_fingerprint (cache_key, payload) VALUES (?, ?)"; +pub(crate) const SELECT_SQL: &str = "SELECT payload FROM migration_fingerprint WHERE cache_key = ?"; + +/// A fingerprint request: query SQL, metadata JSON, dialect and visible renames. +#[derive(Debug, Hash, PartialEq, Eq)] +pub(crate) struct MemoKey { + pub(crate) query_sql: String, + pub(crate) metadata_json: String, + pub(crate) dialect: Option, + pub(crate) visible_renames: Vec<(String, String)>, +} + +static FINGERPRINT_WORK: PyOnceLock> = PyOnceLock::new(); + +fn fingerprint_work(py: Python<'_>) -> PyResult> { + Ok(FINGERPRINT_WORK + .get_or_try_init(py, || Py::new(py, FingerprintWork::default()))? + .bind(py) + .borrow_mut()) +} + +pub(crate) fn count_computation(py: Python<'_>, key: &MemoKey) -> PyResult<()> { + let mut hasher = DefaultHasher::new(); + key.hash(&mut hasher); + let mut work = fingerprint_work(py)?; + work.computations += 1; + work.distinct.insert(hasher.finish()); + Ok(()) +} + +/// Report migration fingerprints computed and distinct requests among them, optionally resetting. +#[pyfunction] +#[pyo3(signature = (*, reset = false))] +pub(crate) fn migration_fingerprint_work_info( + py: Python<'_>, + reset: bool, +) -> PyResult<(u64, usize)> { + let mut work = fingerprint_work(py)?; + let info = (work.computations, work.distinct.len()); + if reset { + *work = FingerprintWork::default(); + } + Ok(info) +} + +pub(crate) fn sqlite_error(py: Python<'_>, error: &PyErr) -> PyResult { + Ok(error.is_instance(py, &imported(py, "sqlite3", "DatabaseError")?)) +} + +pub(crate) fn close(py: Python<'_>, connection: Option>) -> PyResult<()> { + let Some(connection) = connection else { + return Ok(()); + }; + match connection.bind(py).call_method0("close") { + Ok(_) => Ok(()), + Err(error) if sqlite_error(py, &error)? => Ok(()), + Err(error) => Err(error), + } +} + +pub(crate) fn connect<'py>(path: &Bound<'py, PyAny>) -> PyResult> { + let py = path.py(); + call_with( + &imported(py, "sqlite3", "connect")?, + &[ + ("database", path.clone()), + ( + "timeout", + PyFloat::new(py, SQLITE_TIMEOUT_SECONDS).into_any(), + ), + ], + ) +} + +pub(crate) fn package_version(py: Python<'_>, package: &str) -> PyResult { + match imported(py, "importlib.metadata", "version")?.call1((package,)) { + Ok(version) => version.extract(), + Err(error) + if error.is_instance( + py, + &imported(py, "importlib.metadata", "PackageNotFoundError")?, + ) => + { + Ok("unknown".to_owned()) + } + Err(error) => Err(error), + } +} + +/// `migration_fingerprint_work_info` as a function of `module`. +pub(crate) fn migration_fingerprint_work_info_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(migration_fingerprint_work_info, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/mod.rs new file mode 100644 index 0000000000..1c57d002cb --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod cursors; +pub(crate) mod fingerprint_cache; +pub(crate) mod snapshot; +pub(crate) mod watermarks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs index a16d161f3a..74b6241ade 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs @@ -1,11 +1,15 @@ //! Gather a frozen point-in-time warehouse snapshot for planning. -use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; -use crate::bindings::_helpers::planner_warehouse::constants::{ - ACTIVE_CATALOG, FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, MIGRATE_FROM_CONFIG_KEY, - RELATIONS_MAIN_COLUMNS, RELATIONS_MAIN_LIST, -}; -use crate::bindings::_helpers::planner_warehouse::cursors::{CursorInputs, cursor_snapshots}; +use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; +use crate::planner_warehouse::_helpers::inspection::cursors::CursorInputs; +use crate::planner_warehouse::_helpers::inspection::cursors::cursor_snapshots; +use crate::planner_warehouse::constants::ACTIVE_CATALOG; +use crate::planner_warehouse::constants::FUNCTION_NODE_TYPES; +use crate::planner_warehouse::constants::METADATA_NAME_FILTER_LIMIT; +use crate::planner_warehouse::constants::MIGRATE_FROM_CONFIG_KEY; +use crate::planner_warehouse::constants::RELATIONS_MAIN_COLUMNS; +use crate::planner_warehouse::constants::RELATIONS_MAIN_LIST; +use crate::planner_warehouse::models::SnapshotRequest; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -48,20 +52,6 @@ use sqlbuild_python_project::planner_scope::main::build_upstream_intermediate_so use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; use std::collections::{BTreeSet, HashSet}; -/// Everything one snapshot gather needs. -pub(crate) struct SnapshotRequest<'py> { - pub(crate) project: Bound<'py, PyAny>, - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) connection: Bound<'py, PyAny>, - pub(crate) execute: Bound<'py, PyAny>, - pub(crate) selected_keys: Bound<'py, PyAny>, - pub(crate) full_refresh: bool, - pub(crate) full_refresh_model_names: Bound<'py, PyAny>, - pub(crate) on_progress: Bound<'py, PyAny>, - pub(crate) deferred_locations: Bound<'py, PyAny>, - pub(crate) cursor_scope: Bound<'py, PyAny>, -} - #[derive(Default)] struct StateSchemas { fingerprints: BTreeSet, @@ -899,3 +889,17 @@ pub(crate) fn planner_metadata_name_filter<'py>( let selected = selected_keys.unwrap_or_else(|| py.None().into_bound(py)); optional_names(py, metadata_name_filter(project, &selected)?.as_ref()) } + +/// `gather_warehouse_snapshot` as a function of `module`. +pub(crate) fn gather_warehouse_snapshot_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(gather_warehouse_snapshot, module) +} + +/// `planner_metadata_name_filter` as a function of `module`. +pub(crate) fn planner_metadata_name_filter_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(planner_metadata_name_filter, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs index ea9d541ea8..85d7cad00b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs @@ -1,9 +1,9 @@ //! Validate reliable watermark column contracts before warehouse inspection. -use crate::bindings::_helpers::planner_strategy::cursor_types::classify_cursor_sql_type; -use crate::bindings::_helpers::planner_warehouse::constants::{ - CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, WATERMARK_CONTRACT_CODE, -}; +use crate::planner_strategy::main::cursor_types::classify_cursor_sql_type::classify_cursor_sql_type; +use crate::planner_warehouse::constants::CURSOR_TYPE_INTEGER; +use crate::planner_warehouse::constants::CURSOR_TYPE_TIMESTAMP; +use crate::planner_warehouse::constants::WATERMARK_CONTRACT_CODE; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods}; @@ -163,3 +163,10 @@ pub(crate) fn validate_watermark_column<'py>( (Some(WATERMARK_CONTRACT_CODE), None), )) } + +/// `watermark_type_is_compatible` as a function of `module`. +pub(crate) fn watermark_type_is_compatible_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(watermark_type_is_compatible, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/mod.rs new file mode 100644 index 0000000000..349ca8f38c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod inspection; +pub(crate) mod sources; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/mod.rs new file mode 100644 index 0000000000..9ec65b96fd --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod semantic_sources; +pub(crate) mod source_columns; +pub(crate) mod source_deferral; +pub(crate) mod source_freshness; +pub(crate) mod source_tables; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs index d0a0ae05f0..34841887b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs @@ -1,7 +1,8 @@ //! Complete missing source metadata in bounded, database-grouped batches. -use crate::bindings::_helpers::planner_warehouse::constants::ACTIVE_CATALOG; -use crate::bindings::_helpers::planner_warehouse::snapshot::{columns_for, list_relations}; +use crate::planner_warehouse::_helpers::inspection::snapshot::columns_for; +use crate::planner_warehouse::_helpers::inspection::snapshot::list_relations; +use crate::planner_warehouse::constants::ACTIVE_CATALOG; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; @@ -294,3 +295,17 @@ pub(crate) fn semantic_source_columns<'py>( } Ok(result) } + +/// `list_source_candidates` as a function of `module`. +pub(crate) fn list_source_candidates_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(list_source_candidates, module) +} + +/// `semantic_source_columns` as a function of `module`. +pub(crate) fn semantic_source_columns_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(semantic_source_columns, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs index 1047e704e8..5c82c1fb22 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs @@ -1,10 +1,11 @@ //! Warehouse source relation listing and column gathering for planning. -use crate::bindings::_helpers::observations::source_capture::capture_source_columns; -use crate::bindings::_helpers::planner_warehouse::constants::{ - COLUMN_INFO, METADATA_NAME_FILTER_LIMIT, RELATION_LOOKUP, -}; -use crate::bindings::_helpers::planner_warehouse::snapshot::{columns_for, list_relations}; +use crate::observations::main::capture_source_columns::capture_source_columns; +use crate::planner_warehouse::_helpers::inspection::snapshot::columns_for; +use crate::planner_warehouse::_helpers::inspection::snapshot::list_relations; +use crate::planner_warehouse::constants::COLUMN_INFO; +use crate::planner_warehouse::constants::METADATA_NAME_FILTER_LIMIT; +use crate::planner_warehouse::constants::RELATION_LOOKUP; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -248,3 +249,10 @@ pub(crate) fn gather_source_columns<'py>( ) -> PyResult> { Ok(source_inspection(project, (adapter, connection), source_entries)?.0) } + +/// `gather_source_columns` as a function of `module`. +pub(crate) fn gather_source_columns_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(gather_source_columns, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs index 5b0f7ff821..f3e955a49f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs @@ -2,9 +2,8 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::planner_warehouse::constants::{ - MODEL_HOOK_CONFIG_KEYS, SQL_HOOK_ENTRY, -}; +use crate::planner_warehouse::constants::MODEL_HOOK_CONFIG_KEYS; +use crate::planner_warehouse::constants::SQL_HOOK_ENTRY; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs index 5c8143ab3b..1f8797f702 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs @@ -1,15 +1,15 @@ //! Direct planner source freshness observation and the warnings for unknown freshness. -use crate::bindings::_helpers::freshness::planning::PREVIEW_MESSAGE; -use crate::bindings::_helpers::freshness::planning::{ - build_direct_source_freshness_planning_result, build_direct_source_freshness_propagation_result, -}; -use crate::bindings::_helpers::planner_warehouse::constants::RUN_SCOPE; -use crate::bindings::_helpers::planner_warehouse::snapshot::{destination_nodes, lower}; +use crate::freshness::constants::PREVIEW_MESSAGE; +use crate::freshness::main::build_direct_source_freshness_planning_result_function::build_direct_source_freshness_planning_result_function; +use crate::freshness::main::build_direct_source_freshness_propagation_result_function::build_direct_source_freshness_propagation_result_function; +use crate::planner_warehouse::_helpers::inspection::snapshot::destination_nodes; +use crate::planner_warehouse::_helpers::inspection::snapshot::lower; +use crate::planner_warehouse::constants::RUN_SCOPE; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyBool, PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_analysis::planner_warehouse::main::freshness_unknown_message::freshness_unknown_message; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -173,7 +173,7 @@ pub(crate) fn build_planner_source_freshness_result<'py>( &host_entry(py, RUN_SCOPE)?.call1((native_field(project, "run_id")?,))?, |_| { call_with( - wrap_pyfunction!(build_direct_source_freshness_planning_result, py)?.as_any(), + build_direct_source_freshness_planning_result_function(py)?.as_any(), &[ ("adapter", adapter.clone()), ("connection", connection.clone()), @@ -199,7 +199,7 @@ pub(crate) fn build_planner_source_freshness_result<'py>( }, )?; let propagation = call_with( - wrap_pyfunction!(build_direct_source_freshness_propagation_result, py)?.as_any(), + build_direct_source_freshness_propagation_result_function(py)?.as_any(), &[ ("source_freshness", result.clone()), ("scope", scope.clone()), @@ -207,3 +207,10 @@ pub(crate) fn build_planner_source_freshness_result<'py>( )?; replace(&result, &[("propagation", propagation)]) } + +/// `build_planner_source_freshness_result_of` as a function of `module`. +pub(crate) fn build_planner_source_freshness_result_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_planner_source_freshness_result_of, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs index ed0277c229..ba807775a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs @@ -1,8 +1,8 @@ //! Selected-scope validation that external source tables exist in the warehouse. -use crate::bindings::_helpers::planner_warehouse::constants::{ - MISSING_SOURCE_TABLE_CODE, RUN_BOUNDED, WILDCARD_TABLE_CHARACTER, -}; +use crate::planner_warehouse::constants::MISSING_SOURCE_TABLE_CODE; +use crate::planner_warehouse::constants::RUN_BOUNDED; +use crate::planner_warehouse::constants::WILDCARD_TABLE_CHARACTER; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; @@ -238,3 +238,10 @@ pub(crate) fn check_selected_source_tables_exist_binding<'py>( check_selected_source_tables_exist(project, (adapter, connection), scope, relations) } + +/// `check_selected_source_tables_exist_binding` as a function of `module`. +pub(crate) fn check_selected_source_tables_exist_binding_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(check_selected_source_tables_exist_binding, module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs index e684661f10..b251f1b2c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs @@ -1,31 +1,54 @@ -//! Python modules and fixed names of warehouse inspection for planning. - -use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; +//! Public constants of planner warehouse. +use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_python_base::records::types::RecordId; pub(crate) const RELATIONS_MAIN_LIST: AdapterHost = AdapterHost::ListRelationsForInspection; + pub(crate) const RELATIONS_MAIN_COLUMNS: AdapterHost = AdapterHost::GetColumnsForInspection; + pub(crate) const ACTIVE_CATALOG: AdapterHost = AdapterHost::ActiveInspectionCatalog; -pub(crate) const RUN_BOUNDED: AdapterHost = AdapterHost::RunBoundedInspections; + +pub const RUN_BOUNDED: AdapterHost = AdapterHost::RunBoundedInspections; + pub(crate) const RECORD_QUERY: AdapterHost = AdapterHost::RecordInspectionQuery; + pub(crate) const RUN_RECORDED_QUERY: AdapterHost = AdapterHost::RunRecordedInspectionQuery; + pub(crate) const QUERY_RECORD: AdapterHost = AdapterHost::InspectionQueryRecord; + pub(crate) const COLUMN_INFO: AdapterHost = AdapterHost::ColumnInfo; + pub(crate) const RELATION_LOOKUP: AdapterHost = AdapterHost::RelationLookup; + pub(crate) const LOG_DEBUG_EVENT: ExecutorHost = ExecutorHost::LogDebugEvent; + pub(crate) const RUN_SCOPE: ExecutorHost = ExecutorHost::RunScope; + pub(crate) const SQL_HOOK_ENTRY: RecordId = RecordId::SqlHookEntry; + pub(crate) const CURSOR_ALGEBRA_ERROR: (&str, &str) = ("sqlbuild.cursor_algebra.exceptions", "CursorAlgebraError"); + pub(crate) const DATE_VALUE: RecordId = RecordId::DateValue; + pub(crate) const PLANNER_LOGGER: &str = "sqlbuild.planner"; + pub(crate) const METADATA_NAME_FILTER_LIMIT: usize = 250; + pub(crate) const FUNCTION_NODE_TYPES: [&str; 2] = ["udf", "table_fn"]; + pub(crate) const WILDCARD_TABLE_CHARACTER: &str = "*"; + pub(crate) const MIGRATE_FROM_CONFIG_KEY: &str = "migrate_from"; + pub(crate) const MODEL_HOOK_CONFIG_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; + pub(crate) const CURSOR_TYPE_TIMESTAMP: &str = "timestamp"; + pub(crate) const CURSOR_TYPE_INTEGER: &str = "integer"; + pub(crate) const MISSING_SOURCE_TABLE_CODE: &str = "S405"; + pub(crate) const WATERMARK_CONTRACT_CODE: &str = "S302"; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/adapter_cursor_sql.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/adapter_cursor_sql.rs new file mode 100644 index 0000000000..09feaaccf6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/adapter_cursor_sql.rs @@ -0,0 +1,16 @@ +//! The query's MIN/MAX statement, each bound read as text the way the adapter renders it. + +use crate::planner_warehouse::_helpers::inspection::cursors; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::planner_warehouse::models::PhysicalCursorQuery; + +/// The query's MIN/MAX statement, each bound read as text the way the adapter renders it. +#[inline] +pub fn adapter_cursor_sql( + adapter: &Bound<'_, PyAny>, + query: &PhysicalCursorQuery, +) -> PyResult { + cursors::adapter_cursor_sql(adapter, query) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/bound_task.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/bound_task.rs new file mode 100644 index 0000000000..ada17151a7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/bound_task.rs @@ -0,0 +1,17 @@ +//! Bound task. + +use crate::planner_warehouse::_helpers::inspection::cursors; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Bound task. +#[inline] +pub fn bound_task<'py>( + py: Python<'py>, + sql: String, + (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + cursors::bound_task(py, sql, (connection, execute)) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_redirected_cursor_snapshots.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_redirected_cursor_snapshots.rs new file mode 100644 index 0000000000..1108675712 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_redirected_cursor_snapshots.rs @@ -0,0 +1,16 @@ +//! Cursor snapshots of models whose history lives in another relation or column. + +use crate::planner_warehouse::_helpers::inspection::snapshot; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict as KeywordDict; + +/// Cursor snapshots of models whose history lives in another relation or column. +#[inline] +pub fn gather_redirected_cursor_snapshots<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + snapshot::gather_redirected_cursor_snapshots(py, options) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_warehouse_snapshot_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_warehouse_snapshot_function.rs new file mode 100644 index 0000000000..36ea9ef67c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_warehouse_snapshot_function.rs @@ -0,0 +1,11 @@ +//! `gather_warehouse_snapshot` as a function of `module`. + +use crate::planner_warehouse::_helpers::inspection::snapshot; + +/// `gather_warehouse_snapshot` as a function of `module`. +#[inline] +pub fn gather_warehouse_snapshot_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + snapshot::gather_warehouse_snapshot_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/migration_fingerprint_work_info_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/migration_fingerprint_work_info_function.rs new file mode 100644 index 0000000000..82c63e6e98 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/migration_fingerprint_work_info_function.rs @@ -0,0 +1,11 @@ +//! `migration_fingerprint_work_info` as a function of `module`. + +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache; + +/// `migration_fingerprint_work_info` as a function of `module`. +#[inline] +pub fn migration_fingerprint_work_info_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + fingerprint_cache::migration_fingerprint_work_info_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/mod.rs new file mode 100644 index 0000000000..b6ff3dc93c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/mod.rs @@ -0,0 +1,8 @@ +pub mod adapter_cursor_sql; +pub mod bound_task; +pub mod gather_redirected_cursor_snapshots; +pub mod gather_warehouse_snapshot_function; +pub mod migration_fingerprint_work_info_function; +pub mod planner_metadata_name_filter_function; +pub mod warehouse_snapshot; +pub mod watermark_type_is_compatible_function; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/planner_metadata_name_filter_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/planner_metadata_name_filter_function.rs new file mode 100644 index 0000000000..ec24876457 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/planner_metadata_name_filter_function.rs @@ -0,0 +1,11 @@ +//! `planner_metadata_name_filter` as a function of `module`. + +use crate::planner_warehouse::_helpers::inspection::snapshot; + +/// `planner_metadata_name_filter` as a function of `module`. +#[inline] +pub fn planner_metadata_name_filter_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + snapshot::planner_metadata_name_filter_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/warehouse_snapshot.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/warehouse_snapshot.rs new file mode 100644 index 0000000000..9b09c4c861 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/warehouse_snapshot.rs @@ -0,0 +1,13 @@ +//! Gather relations, columns, fingerprints and cursor bounds of the planned target schemas. + +use crate::planner_warehouse::_helpers::inspection::snapshot; +use crate::planner_warehouse::models::SnapshotRequest; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Gather relations, columns, fingerprints and cursor bounds of the planned target schemas. +#[inline] +pub fn warehouse_snapshot<'py>(request: &SnapshotRequest<'py>) -> PyResult> { + snapshot::warehouse_snapshot(request) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/watermark_type_is_compatible_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/watermark_type_is_compatible_function.rs new file mode 100644 index 0000000000..16e7ed3167 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/watermark_type_is_compatible_function.rs @@ -0,0 +1,11 @@ +//! `watermark_type_is_compatible` as a function of `module`. + +use crate::planner_warehouse::_helpers::inspection::watermarks; + +/// `watermark_type_is_compatible` as a function of `module`. +#[inline] +pub fn watermark_type_is_compatible_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + watermarks::watermark_type_is_compatible_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/mod.rs new file mode 100644 index 0000000000..87c68376c1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/mod.rs @@ -0,0 +1,2 @@ +pub mod inspection; +pub mod sources; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result.rs new file mode 100644 index 0000000000..e791a07726 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result.rs @@ -0,0 +1,22 @@ +//! The source freshness evaluation of one inspection scope against the warehouse. + +use crate::planner_warehouse::_helpers::sources::source_freshness; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// The source freshness evaluation of one inspection scope against the warehouse. +#[inline] +pub fn build_planner_source_freshness_result<'py>( + (project, scope): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + relations: &Bound<'py, PyAny>, + freshness_state_schemas: &Bound<'py, PyAny>, +) -> PyResult> { + source_freshness::build_planner_source_freshness_result( + (project, scope), + (adapter, connection), + relations, + freshness_state_schemas, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result_of_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result_of_function.rs new file mode 100644 index 0000000000..96617b10f0 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result_of_function.rs @@ -0,0 +1,11 @@ +//! `build_planner_source_freshness_result_of` as a function of `module`. + +use crate::planner_warehouse::_helpers::sources::source_freshness; + +/// `build_planner_source_freshness_result_of` as a function of `module`. +#[inline] +pub fn build_planner_source_freshness_result_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + source_freshness::build_planner_source_freshness_result_of_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_source_read_map.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_source_read_map.rs new file mode 100644 index 0000000000..42f6381727 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_source_read_map.rs @@ -0,0 +1,16 @@ +//! Resolve source reads for selected SQL, or for `read_source_names` when given. + +use crate::planner_warehouse::_helpers::sources::source_deferral; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyDict as KeywordDict; + +/// Resolve source reads for selected SQL, or for `read_source_names` when given. +#[inline] +pub fn build_source_read_map<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + source_deferral::build_source_read_map(py, options) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist.rs new file mode 100644 index 0000000000..c25e455328 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist.rs @@ -0,0 +1,22 @@ +//! Raise S405 when a selected resource reads an external source table that does not exist. + +use crate::planner_warehouse::_helpers::sources::source_tables; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Raise S405 when a selected resource reads an external source table that does not exist. +#[inline] +pub fn check_selected_source_tables_exist<'py>( + project: &Bound<'py, PyAny>, + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, +) -> PyResult<()> { + source_tables::check_selected_source_tables_exist( + project, + (adapter, connection), + scope, + relations, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist_binding_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist_binding_function.rs new file mode 100644 index 0000000000..d8248caa1f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist_binding_function.rs @@ -0,0 +1,11 @@ +//! `check_selected_source_tables_exist_binding` as a function of `module`. + +use crate::planner_warehouse::_helpers::sources::source_tables; + +/// `check_selected_source_tables_exist_binding` as a function of `module`. +#[inline] +pub fn check_selected_source_tables_exist_binding_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + source_tables::check_selected_source_tables_exist_binding_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns.rs new file mode 100644 index 0000000000..3947717f6e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns.rs @@ -0,0 +1,18 @@ +//! Gather source columns. + +use crate::planner_warehouse::_helpers::sources::source_columns; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Gather source columns. +#[inline] +pub fn gather_source_columns<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + source_entries: Option>, +) -> PyResult> { + source_columns::gather_source_columns(project, adapter, connection, source_entries) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns_function.rs new file mode 100644 index 0000000000..76ae5e4d3a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns_function.rs @@ -0,0 +1,11 @@ +//! `gather_source_columns` as a function of `module`. + +use crate::planner_warehouse::_helpers::sources::source_columns; + +/// `gather_source_columns` as a function of `module`. +#[inline] +pub fn gather_source_columns_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + source_columns::gather_source_columns_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_inspection.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_inspection.rs new file mode 100644 index 0000000000..0153d0510a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_inspection.rs @@ -0,0 +1,17 @@ +//! Gather source inspection. + +use crate::planner_warehouse::_helpers::sources::source_columns; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// Gather source inspection. +#[inline] +pub fn gather_source_inspection<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + source_entries: Option>, +) -> PyResult> { + source_columns::gather_source_inspection(project, adapter, connection, source_entries) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/list_source_candidates_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/list_source_candidates_function.rs new file mode 100644 index 0000000000..bd40139eaa --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/list_source_candidates_function.rs @@ -0,0 +1,11 @@ +//! `list_source_candidates` as a function of `module`. + +use crate::planner_warehouse::_helpers::sources::semantic_sources; + +/// `list_source_candidates` as a function of `module`. +#[inline] +pub fn list_source_candidates_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + semantic_sources::list_source_candidates_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/mod.rs new file mode 100644 index 0000000000..2ec99c7e4a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/mod.rs @@ -0,0 +1,13 @@ +pub mod build_planner_source_freshness_result; +pub mod build_planner_source_freshness_result_of_function; +pub mod build_source_read_map; +pub mod check_selected_source_tables_exist; +pub mod check_selected_source_tables_exist_binding_function; +pub mod gather_source_columns; +pub mod gather_source_columns_function; +pub mod gather_source_inspection; +pub mod list_source_candidates_function; +pub mod semantic_source_columns; +pub mod semantic_source_columns_function; +pub mod source_freshness_unknown_warnings; +pub mod with_declared_source_reads; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns.rs new file mode 100644 index 0000000000..71750cb10b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns.rs @@ -0,0 +1,16 @@ +//! Reuse collected columns and inspect only selected, unresolved table sources. + +use crate::planner_warehouse::_helpers::sources::semantic_sources; +use pyo3::prelude::Bound; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; +use pyo3::types::PyDict as KeywordDict; + +/// Reuse collected columns and inspect only selected, unresolved table sources. +#[inline] +pub fn semantic_source_columns<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + semantic_sources::semantic_source_columns(py, options) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns_function.rs new file mode 100644 index 0000000000..9579b5c5fb --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns_function.rs @@ -0,0 +1,11 @@ +//! `semantic_source_columns` as a function of `module`. + +use crate::planner_warehouse::_helpers::sources::semantic_sources; + +/// `semantic_source_columns` as a function of `module`. +#[inline] +pub fn semantic_source_columns_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + semantic_sources::semantic_source_columns_function(module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/source_freshness_unknown_warnings.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/source_freshness_unknown_warnings.rs new file mode 100644 index 0000000000..ffdb9e6ece --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/source_freshness_unknown_warnings.rs @@ -0,0 +1,14 @@ +//! One warning per reason that left configured source freshness unknown. + +use crate::planner_warehouse::_helpers::sources::source_freshness; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; + +/// One warning per reason that left configured source freshness unknown. +#[inline] +pub fn source_freshness_unknown_warnings<'py>( + source_freshness: &Bound<'py, PyAny>, +) -> PyResult> { + source_freshness::source_freshness_unknown_warnings(source_freshness) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/with_declared_source_reads.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/with_declared_source_reads.rs new file mode 100644 index 0000000000..6be27897b5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/with_declared_source_reads.rs @@ -0,0 +1,17 @@ +//! Add declared project sources that no selected SQL resource reads to a source map. + +use crate::planner_warehouse::_helpers::sources::source_deferral; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::types::PyDict; + +/// Add declared project sources that no selected SQL resource reads to a source map. +#[inline] +pub fn with_declared_source_reads<'py>( + project: &Bound<'py, PyAny>, + source_map: &Bound<'py, PyDict>, + source_names: &Bound<'py, PyAny>, +) -> PyResult> { + source_deferral::with_declared_source_reads(project, source_map, source_names) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/mod.rs new file mode 100644 index 0000000000..106e3b1428 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/mod.rs @@ -0,0 +1,6 @@ +//! Native warehouse inspection for planning over Python adapter callbacks. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs similarity index 67% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/models.rs index d5c1e961ed..fc106c2d09 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs @@ -1,137 +1,63 @@ -//! Migration fingerprints computed at most once per model definition and reused across runs. +//! Public records of planner warehouse. -use crate::bindings::_helpers::planner_changes::metadata::without_declared_columns_hash; -use crate::bindings::_helpers::planner_migrations::fingerprint::fingerprint; -use crate::bindings::_helpers::planner_migrations::fingerprint::migration_fingerprint_ref_names; -use crate::bindings::_helpers::planner_warehouse::models::FingerprintWork; -use pyo3::PyRefMut; -use pyo3::exceptions::{PyOSError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyBool, PyDict, PyFloat, PyList, PyListMethods, PyTuple}; -use pyo3::{pyclass, pyfunction, pymethods}; +use crate::planner_changes::main::without_declared_columns_hash::without_declared_columns_hash; +use crate::planner_migrations::main::fingerprint::fingerprint; +use crate::planner_migrations::main::migration_fingerprint_ref_names::migration_fingerprint_ref_names; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::CACHE_DATABASE_NAME; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::CACHE_DIRECTORY_NAME; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::CREATE_CACHE_TABLE_SQL; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::INSERT_SQL; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::MemoKey; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::SELECT_SQL; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::close; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::connect; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::count_computation; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::package_version; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::sqlite_error; +use pyo3::exceptions::PyOSError; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyBool; +use pyo3::types::PyDict; +use pyo3::types::PyList; +use pyo3::types::PyListMethods; +use pyo3::types::PyTuple; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_entry::fingerprint_cache_entry; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_key::fingerprint_cache_key; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_lookup::fingerprint_cache_lookup; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_shared_digest::fingerprint_cache_shared_digest; use sqlbuild_analysis::planner_warehouse::models::FingerprintCacheKey; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_project::planner_phases::main::object::object; -use std::collections::{BTreeMap, HashMap}; -use std::hash::{DefaultHasher, Hash, Hasher}; - -const CACHE_DIRECTORY_NAME: &str = "migration-fingerprints-v1"; -const CACHE_DATABASE_NAME: &str = "migration-fingerprints.sqlite3"; -const SQLITE_TIMEOUT_SECONDS: f64 = 0.1; -const CREATE_CACHE_TABLE_SQL: &str = "\nCREATE TABLE IF NOT EXISTS migration_fingerprint (\n cache_key TEXT PRIMARY KEY,\n payload TEXT NOT NULL\n)\n"; -const INSERT_SQL: &str = - "INSERT OR REPLACE INTO migration_fingerprint (cache_key, payload) VALUES (?, ?)"; -const SELECT_SQL: &str = "SELECT payload FROM migration_fingerprint WHERE cache_key = ?"; - -/// A fingerprint request: query SQL, metadata JSON, dialect and visible renames. -#[derive(Hash, PartialEq, Eq)] -struct MemoKey { - query_sql: String, - metadata_json: String, - dialect: Option, - visible_renames: Vec<(String, String)>, -} - -static FINGERPRINT_WORK: PyOnceLock> = PyOnceLock::new(); - -fn fingerprint_work(py: Python<'_>) -> PyResult> { - Ok(FINGERPRINT_WORK - .get_or_try_init(py, || Py::new(py, FingerprintWork::default()))? - .bind(py) - .borrow_mut()) -} - -fn count_computation(py: Python<'_>, key: &MemoKey) -> PyResult<()> { - let mut hasher = DefaultHasher::new(); - key.hash(&mut hasher); - let mut work = fingerprint_work(py)?; - work.computations += 1; - work.distinct.insert(hasher.finish()); - Ok(()) -} - -/// Report migration fingerprints computed and distinct requests among them, optionally resetting. -#[pyfunction] -#[pyo3(signature = (*, reset = false))] -pub(crate) fn migration_fingerprint_work_info( - py: Python<'_>, - reset: bool, -) -> PyResult<(u64, usize)> { - let mut work = fingerprint_work(py)?; - let info = (work.computations, work.distinct.len()); - if reset { - *work = FingerprintWork::default(); - } - Ok(info) -} +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::collections::HashMap; /// Reuse migration fingerprints across rename maps, planning phases and invocations. #[pyclass(module = "sqlbuild._native")] -pub(crate) struct MigrationFingerprintCache { - ref_names: HashMap<(String, String), Vec>, - fingerprints: HashMap>, - database_path: Option>, - connection: Option>, - connection_opened: bool, - pending: Vec<(String, String)>, - shared_digest: String, -} - -fn sqlite_error(py: Python<'_>, error: &PyErr) -> PyResult { - Ok(error.is_instance(py, &imported(py, "sqlite3", "DatabaseError")?)) -} - -fn close(py: Python<'_>, connection: Option>) -> PyResult<()> { - let Some(connection) = connection else { - return Ok(()); - }; - match connection.bind(py).call_method0("close") { - Ok(_) => Ok(()), - Err(error) if sqlite_error(py, &error)? => Ok(()), - Err(error) => Err(error), - } -} - -fn connect<'py>(path: &Bound<'py, PyAny>) -> PyResult> { - let py = path.py(); - call_with( - &imported(py, "sqlite3", "connect")?, - &[ - ("database", path.clone()), - ( - "timeout", - PyFloat::new(py, SQLITE_TIMEOUT_SECONDS).into_any(), - ), - ], - ) -} - -fn package_version(py: Python<'_>, package: &str) -> PyResult { - match imported(py, "importlib.metadata", "version")?.call1((package,)) { - Ok(version) => version.extract(), - Err(error) - if error.is_instance( - py, - &imported(py, "importlib.metadata", "PackageNotFoundError")?, - ) => - { - Ok("unknown".to_owned()) - } - Err(error) => Err(error), - } +#[derive(Debug)] +pub struct MigrationFingerprintCache { + pub(crate) ref_names: HashMap<(String, String), Vec>, + pub(crate) fingerprints: HashMap>, + pub(crate) database_path: Option>, + pub(crate) connection: Option>, + pub(crate) connection_opened: bool, + pub(crate) pending: Vec<(String, String)>, + pub(crate) shared_digest: String, } impl MigrationFingerprintCache { /// A cache persisting under `root`, or an in-memory cache without one. - pub(crate) fn open(py: Python<'_>, root: Option>) -> PyResult { + pub fn open(py: Python<'_>, root: Option>) -> PyResult { let root = root.filter(|root| !root.is_none()); let database_path = match &root { Some(root) => Some( @@ -161,7 +87,7 @@ impl MigrationFingerprintCache { }) } - fn cache_key(&self, key: &MemoKey) -> PyResult { + pub(crate) fn cache_key(&self, key: &MemoKey) -> PyResult { if self.database_path.is_none() { return Ok(String::new()); } @@ -175,7 +101,7 @@ impl MigrationFingerprintCache { .map_err(|error| PyValueError::new_err(format!("{error:?}"))) } - fn open_connection(&mut self, py: Python<'_>) -> PyResult>> { + pub(crate) fn open_connection(&mut self, py: Python<'_>) -> PyResult>> { if self.connection_opened { return Ok(self .connection @@ -203,7 +129,11 @@ impl MigrationFingerprintCache { .map(|connection| connection.clone_ref(py))) } - fn stored(&mut self, py: Python<'_>, cache_key: &str) -> PyResult>> { + pub(crate) fn stored( + &mut self, + py: Python<'_>, + cache_key: &str, + ) -> PyResult>> { let Some(connection) = self.open_connection(py)? else { return Ok(None); }; @@ -230,7 +160,7 @@ impl MigrationFingerprintCache { Ok(fingerprint_cache_lookup(&contents, cache_key)) } - fn remember(&mut self, cache_key: &str, fingerprint: Option<&str>) -> PyResult<()> { + pub(crate) fn remember(&mut self, cache_key: &str, fingerprint: Option<&str>) -> PyResult<()> { if self.database_path.is_none() { return Ok(()); } @@ -243,7 +173,7 @@ impl MigrationFingerprintCache { Ok(()) } - fn write_pending( + pub(crate) fn write_pending( &self, py: Python<'_>, pending: &[(String, String)], @@ -297,13 +227,13 @@ impl MigrationFingerprintCache { impl MigrationFingerprintCache { #[new] #[pyo3(signature = (*, root=None))] - fn new(py: Python<'_>, root: Option>) -> PyResult { + pub(crate) fn new(py: Python<'_>, root: Option>) -> PyResult { Self::open(py, root) } /// The migration fingerprint, keyed only by the renames this model can see. #[pyo3(signature = (**options))] - fn fingerprint( + pub(crate) fn fingerprint( &mut self, py: Python<'_>, options: Option>, @@ -377,7 +307,7 @@ impl MigrationFingerprintCache { } /// Write fingerprints computed since the last persist; failures only skip reuse. - fn persist(&mut self, py: Python<'_>) -> PyResult<()> { + pub(crate) fn persist(&mut self, py: Python<'_>) -> PyResult<()> { let pending = std::mem::take(&mut self.pending); let connection = self.connection.take(); self.connection_opened = false; @@ -391,3 +321,26 @@ impl MigrationFingerprintCache { closed } } + +/// Migration fingerprints computed since the last reset, for work-count tests. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct FingerprintWork { + pub computations: u64, + pub distinct: BTreeSet, +} + +/// Everything one snapshot gather needs. +#[derive(Debug)] +pub struct SnapshotRequest<'py> { + pub project: Bound<'py, PyAny>, + pub adapter: Bound<'py, PyAny>, + pub connection: Bound<'py, PyAny>, + pub execute: Bound<'py, PyAny>, + pub selected_keys: Bound<'py, PyAny>, + pub full_refresh: bool, + pub full_refresh_model_names: Bound<'py, PyAny>, + pub on_progress: Bound<'py, PyAny>, + pub deferred_locations: Bound<'py, PyAny>, + pub cursor_scope: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 75424c5112..7e7f354170 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -31,7 +31,6 @@ sqlbuild-cache.workspace = true sqlbuild-cli.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true -sqlbuild-cursor.workspace = true sqlbuild-discovery.workspace = true sqlbuild-host.workspace = true sqlbuild-model-config.workspace = true @@ -41,6 +40,7 @@ sqlbuild-pysource.workspace = true sqlbuild-python-base.workspace = true sqlbuild-python-front.workspace = true sqlbuild-python-project.workspace = true +sqlbuild-python-planner.workspace = true sqlbuild-recall.workspace = true sqlbuild-refactor.workspace = true sqlbuild-render.workspace = true @@ -49,7 +49,6 @@ sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlbuild-state.workspace = true sqlclean.workspace = true -uuid.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 05d1c80c5f..f7224beb48 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -23,7 +23,6 @@ use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_com use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::planner_entries::command_plans::build_audit_command_plan; use crate::bindings::_helpers::result_output::documents::{ audit_execution_json, audit_projection, write_execution_json_output, }; @@ -42,6 +41,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; use sqlbuild_python_front::cost_context::models::CostContext; +use sqlbuild_python_planner::planner_entries::main::plans::build_audit_command_plan::build_audit_command_plan; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; use sqlbuild_python_project::project_snapshot::types::VerifiedRun; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 5faa65a20e..7ec58f3fd5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -19,9 +19,6 @@ use crate::bindings::_helpers::build_command::planning::{ write_compile_target, write_plan_text, }; use crate::bindings::_helpers::check::check_core::relevant_check_functions; -use crate::bindings::_helpers::observations::source_capture::{ - arm_source_capture, take_source_capture, -}; use crate::bindings::_helpers::recall::console::write_stderr; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; @@ -34,6 +31,8 @@ use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_use use sqlbuild_python_front::discovery::main::sources::provider_uses::provider_uses; use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; +use sqlbuild_python_planner::observations::main::arm_source_capture::arm_source_capture; +use sqlbuild_python_planner::observations::main::take_source_capture::take_source_capture; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index ddce14bdbd..b013e1b08d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -27,13 +27,6 @@ use crate::bindings::_helpers::check::check_core::{ use crate::bindings::_helpers::check::check_results::write_check_results; use crate::bindings::_helpers::command_runtime::warehouses::describe_connection_warehouse; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; -use crate::bindings::_helpers::consumption::policy::{changed_policy, policy_line}; -use crate::bindings::_helpers::consumption::recorder::{ - ConsumptionRecorder, Prepared, held_back_text, prepare as prepare_consumption, run_reason_text, -}; -use crate::bindings::_helpers::freshness::publication::append_freshness_records; -use crate::bindings::_helpers::observations::publication::append_build_observations; -use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::sql_test_command::test_command::timeout_record; use crate::bindings::_helpers::sql_test_command::timeouts::{TimeoutKind, resolve_run_timeout}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -59,6 +52,16 @@ use sqlbuild_python_front::cost_context::models::CostContext; use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; use sqlbuild_python_front::native_host::main::recorder_callbacks::recorder_callbacks; +use sqlbuild_python_planner::consumption::main::changed_policy::changed_policy; +use sqlbuild_python_planner::consumption::main::held_back_text::held_back_text; +use sqlbuild_python_planner::consumption::main::policy_line::policy_line; +use sqlbuild_python_planner::consumption::main::prepare::prepare as prepare_consumption; +use sqlbuild_python_planner::consumption::main::run_reason_text::run_reason_text; +use sqlbuild_python_planner::consumption::models::ConsumptionRecorder; +use sqlbuild_python_planner::consumption::models::Prepared; +use sqlbuild_python_planner::freshness::main::append_freshness_records::append_freshness_records; +use sqlbuild_python_planner::observations::main::append_build_observations::append_build_observations; +use sqlbuild_python_planner::planner_resolution::main::cursor::type_cursor_overrides::type_cursor_overrides; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_state::consumption::models::ChangedPolicy; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index fb29fddc1f..462324401a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -15,7 +15,6 @@ use crate::bindings::_helpers::build_progress::progress_text::with_verbose_inspe use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::manifest::builder::build_manifest; -use crate::bindings::_helpers::plan_output::plan_text::format_plan; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::build_command::main::boolean::boolean; @@ -27,6 +26,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_planner::plan_output::main::format_plan::format_plan; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs index 625fb38bf1..915ef8e1f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs @@ -10,8 +10,8 @@ use sqlbuild_core::plan_display::main::model_resource_type::model_resource_type; use sqlbuild_core::plan_display::models::ModelLabelFacts; use crate::bindings::_helpers::build_progress::result_text::text; -use crate::bindings::_helpers::plan_output::display::optional_facts; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_planner::plan_output::main::optional_facts::optional_facts; const SOURCE_TYPE: &str = "source"; const SQL_TEST_TYPE: &str = "sql_test"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs index 6009f9bdc0..f5b61e58f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs @@ -25,10 +25,11 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::plan_output::display::{cursor_bound_display, inclusive_cursor_end}; use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::result_output::main::json_value::json_value; +use sqlbuild_python_planner::plan_output::main::cursor_bound_display::cursor_bound_display; +use sqlbuild_python_planner::plan_output::main::inclusive_cursor_end::inclusive_cursor_end; const DECIMAL_KIND: &str = "decimal"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs index 7cf4d4902f..f44cf2dc69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs @@ -21,8 +21,9 @@ use sqlbuild_analysis::definition_changes::models::{DefinitionSnapshot, NodeDefi use crate::bindings::_helpers::change_selection::fingerprints::{ Fingerprinting, directory_digest, file_digest, text_of, }; -use crate::bindings::_helpers::planner_identity::metadata::{function_local_hashes, seed_identity}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_planner::planner_identity::main::function_local_hashes::function_local_hashes; +use sqlbuild_python_planner::planner_identity::main::seed_identity::seed_identity; use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs index 293c886880..92bab8e8e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs @@ -26,9 +26,9 @@ use crate::bindings::_helpers::change_selection::definition_snapshot::{ declaration_uses, definition_snapshot, project_wide_keys, }; use crate::bindings::_helpers::change_selection::environment_nodes::with_current_dependencies; -use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; use crate::bindings::_helpers::scenario_command::setup::{flush, write}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_planner::planner_scenario::main::plan_scenario_graph::plan_scenario_graph; use sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 14347b476b..71aab74403 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -8,7 +8,6 @@ use crate::bindings::_helpers::compile_driver::relation_targets::build_project_r use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::planner_entries::command_plans::build_relation_command_plan; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -30,6 +29,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; +use sqlbuild_python_planner::planner_entries::main::plans::build_relation_command_plan::build_relation_command_plan; use sqlbuild_python_project::graph::main::nodes::sql_ref_key::sql_ref_key; use sqlbuild_python_project::graph::main::nodes::transitive_closure_many::transitive_closure_many; use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index 4fd56ca394..4809d49ef6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -28,7 +28,6 @@ use crate::bindings::_helpers::clone::clone_progress::{clone_item_text, write_li use crate::bindings::_helpers::command_runtime::connection::resolve_target_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::planner_entries::clone_plans::PhaseTimings; use crate::bindings::_helpers::result_output::documents::format_clone_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -47,6 +46,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; +use sqlbuild_python_planner::planner_entries::types::PhaseTimings; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; const CLONE_LOGGER: &str = "sqlbuild.clone"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs index 184caf11c2..067fdb3917 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs @@ -5,13 +5,14 @@ use pyo3::types::{PyBool, PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::planner_entries::clone_plans::{PhaseTimings, timed_clone_planning}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_planner::planner_entries::main::plans::timed_clone_planning::timed_clone_planning; +use sqlbuild_python_planner::planner_entries::types::PhaseTimings; /// The phase name of one target compile, before the target name. const COMPILE_PHASE_PREFIX: &str = "compile_"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 7cdd47a44b..22c26a8fa0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -9,10 +9,10 @@ use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_g use crate::bindings::_helpers::compile_driver::python_plan_entries::{ build_python_run_plan_outputs, python_read_source_names_for, }; -use crate::bindings::_helpers::planner_entries::execution_plan::build_execution_plan; use sqlbuild_analysis::planner_scope::constants::SOURCE; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_planner::planner_entries::main::plans::build_execution_plan::build_execution_plan; use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::graph::models::KeyCache; use sqlbuild_python_project::planner_phases::main::object::object; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs index 12cc3c0e8e..446d4abf23 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs @@ -10,14 +10,13 @@ use crate::bindings::_helpers::compile_driver::deferral::{ use crate::bindings::_helpers::compile_driver::pipeline::compile_project_phase; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; use crate::bindings::_helpers::compile_driver::selection_phases::unit_test_selectors_applied; -use crate::bindings::_helpers::planner_entries::command_plans::{ - resolve_static_command_scope, resolve_static_relation_context, -}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_planner::planner_entries::main::plans::resolve_static_command_scope::resolve_static_command_scope; +use sqlbuild_python_planner::planner_entries::main::plans::resolve_static_relation_context::resolve_static_relation_context; use sqlbuild_python_project::graph::main::python_nodes::run_selection_from_inputs::run_selection_from_inputs; /// Locations of the deferred target's relations, when the options defer to one. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs index 509bfa4aca..b0968bf304 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs @@ -8,7 +8,6 @@ use sqlbuild_core::cli_text::main::format_dbt_interop_plan::format_dbt_interop_p use sqlbuild_core::cli_text::main::format_dbt_interop_plan_json::format_dbt_interop_plan_json; use sqlbuild_core::cli_text::models::{DbtInteropPlanText, DbtLsNodeText, DisplayLimit}; -use crate::bindings::_helpers::plan_output::plan_text::format_plan; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; @@ -18,6 +17,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_planner::plan_output::main::format_plan::format_plan; const BUILD_OPTIONS: (&[&str], &[&str]) = ( &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 3ba6d26f14..69c050065a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -11,8 +11,6 @@ use crate::bindings::_helpers::command_runtime::connection::{ use crate::bindings::_helpers::command_runtime::warehouses::apply_command_warehouse; use crate::bindings::_helpers::compile_driver::deferral::resolve_model_deferral_inputs; use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; -use crate::bindings::_helpers::planner_entries::display_plan::build_display_only_sqlbuild_plan; -use crate::bindings::_helpers::planner_entries::execution_plan::build_execution_plan; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -28,6 +26,8 @@ use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; +use sqlbuild_python_planner::planner_entries::main::plans::build_display_only_sqlbuild_plan::build_display_only_sqlbuild_plan; +use sqlbuild_python_planner::planner_entries::main::plans::build_execution_plan::build_execution_plan; const BUILD_OPTIONS: (&[&str], &[&str]) = (&["environment", "request", "hooks"], &[]); const ATTACH_OPTIONS: (&[&str], &[&str]) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 91312c9289..c58a5b69ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -17,9 +17,6 @@ use sqlbuild_orchestration::runs::models::NodeOutcome; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::freshness::freshness_observe::{PreviousState, observe_freshness}; -use crate::bindings::_helpers::freshness::observation::Observer; -use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; @@ -27,6 +24,11 @@ use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_planner::freshness::main::observe_freshness::observe_freshness; +use sqlbuild_python_planner::freshness::main::read_latest::read_latest; +use sqlbuild_python_planner::freshness::models::Observer; +use sqlbuild_python_planner::freshness::models::PreviousState; +use sqlbuild_python_planner::freshness::models::StateRead; use sqlbuild_python_project::planner_selection::main::expand_project_upstream_keys::expand_project_upstream_keys; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs index 98a0933a17..e5f2b2a5fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs @@ -1,9 +1,2 @@ -pub(crate) mod exception_types; pub(crate) mod freshness_command; -pub(crate) mod freshness_observe; -pub(crate) mod observation; -pub(crate) mod planning; -pub(crate) mod publication; pub(crate) mod registration; -pub(crate) mod state_io; -pub(crate) mod support; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs index d81d40f9e0..e69c8470a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs @@ -2,15 +2,18 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::freshness::{ - exception_types, freshness_observe, observation, planning, publication, state_io, -}; +use sqlbuild_python_planner::freshness::main::register_exception_types::register_exception_types; +use sqlbuild_python_planner::freshness::main::register_freshness_observe::register_freshness_observe; +use sqlbuild_python_planner::freshness::main::register_observation::register_observation; +use sqlbuild_python_planner::freshness::main::register_planning::register_planning; +use sqlbuild_python_planner::freshness::main::register_publication::register_publication; +use sqlbuild_python_planner::freshness::main::register_state_io::register_state_io; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - exception_types::register(module)?; - freshness_observe::register(module)?; - observation::register(module)?; - planning::register(module)?; - publication::register(module)?; - state_io::register(module) + register_exception_types(module)?; + register_freshness_observe(module)?; + register_observation(module)?; + register_planning(module)?; + register_publication(module)?; + register_state_io(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs index b36f00df8e..04c11cd605 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs @@ -25,8 +25,8 @@ use crate::bindings::_helpers::janitor::janitor_keys::{ Folded, Key, SchemaKey, archive_timestamp, days, folded_schema, optional, record, relation_age_timestamp, sorted_schemas, text, timestamp_fields, tuple, utc_datetime, }; -use crate::bindings::_helpers::planner_scenario::artifacts::is_scenario_artifact_physical_name; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_planner::planner_scenario::main::is_scenario_artifact_physical_name::is_scenario_artifact_physical_name; use sqlbuild_python_project::relation_naming::main::shared_error::shared_error; /// Protection rules for one plan. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs index ccd5f51ee1..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs @@ -1,5 +1 @@ -pub(crate) mod catalog; -pub(crate) mod envelopes; -pub(crate) mod payloads; pub(crate) mod registration; -pub(crate) mod validation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs index 1bb8d62067..d1cb5c0a9f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs @@ -2,10 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::lifecycle_events::{catalog, envelopes, validation}; +use sqlbuild_python_planner::lifecycle_events::main::register_catalog::register_catalog; +use sqlbuild_python_planner::lifecycle_events::main::register_envelopes::register_envelopes; +use sqlbuild_python_planner::lifecycle_events::main::register_validation::register_validation; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - catalog::register(module)?; - envelopes::register(module)?; - validation::register(module) + register_catalog(module)?; + register_envelopes(module)?; + register_validation(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index a1d82374fb..046277565e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -28,7 +28,6 @@ use crate::bindings::_helpers::load::load_progress::{ use crate::bindings::_helpers::load::load_selection::{ LoadProject, LoaderDefaults, loader_defaults, loader_write_schemas, target_vars, }; -use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::result_output::documents::format_load_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; @@ -52,6 +51,7 @@ use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_use use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; use sqlbuild_python_front::discovery::models::SecretPolicy; +use sqlbuild_python_planner::planner_resolution::main::cursor::type_cursor_overrides::type_cursor_overrides; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 879f3acf19..2f165dfbf8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -16,7 +16,6 @@ use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; -use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::orchestration_reads::main::environment_hash::environment_hash; use sqlbuild_python_base::records::main::fields::field::field; @@ -24,6 +23,7 @@ use sqlbuild_python_front::discovery::main::project::named_node_rows::named_node use sqlbuild_python_front::discovery::main::project::project_facts::project_facts; use sqlbuild_python_front::discovery::main::project::validation_error::validation_error; use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; +use sqlbuild_python_planner::planner_identity::main::direct_model_version_identities::direct_model_version_identities; use sqlbuild_python_project::compiled_views::models::NativeProjectView; use sqlbuild_python_project::graph::models::ProjectSelection; use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/mod.rs index e1b436ef3d..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/mod.rs @@ -1,4 +1 @@ -pub(crate) mod conversion; -pub(crate) mod projection; pub(crate) mod registration; -pub(crate) mod statements; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/registration.rs index b114a939df..73e1e8c28a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::microbatches::{projection, statements}; +use sqlbuild_python_planner::microbatches::main::register_projection::register_projection; +use sqlbuild_python_planner::microbatches::main::register_statements::register_statements; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - projection::register(module)?; - statements::register(module) + register_projection(module)?; + register_statements(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 9ad0e502dd..e02a4eae34 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -20,7 +20,6 @@ pub(crate) mod compile_outputs; pub(crate) mod compiled_project; pub(crate) mod compiled_views; pub(crate) mod compute_logs; -pub(crate) mod consumption; pub(crate) mod contract_command; pub(crate) mod contracts; pub(crate) mod cost_command; @@ -57,7 +56,6 @@ pub(crate) mod manifest; pub(crate) mod microbatches; pub(crate) mod model_config; pub(crate) mod native_host; -pub(crate) mod observations; pub(crate) mod output_capture; pub(crate) mod plan_command; pub(crate) mod plan_output; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 61e4e26117..29d7f8f6dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -34,8 +34,6 @@ use sqlbuild_state::state_tables::main::read_history_sql::read_history_sql; use sqlbuild_state::state_tables::models::HistoryRead; use crate::bindings::_helpers::command_runtime::connection::project_connection_config; -use crate::bindings::_helpers::freshness::state_io::row_record; -use crate::bindings::_helpers::freshness::support::presumed_utc_datetime; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; @@ -49,6 +47,8 @@ use sqlbuild_python_base::warehouse_access::main::read_catalog::read_catalog; use sqlbuild_python_base::warehouse_access::main::read_statistics::read_statistics; use sqlbuild_python_front::authored_values::main::sql_value::sql_value; use sqlbuild_python_front::authored_values::models::SqlValueClasses; +use sqlbuild_python_planner::freshness::main::presumed_utc_datetime::presumed_utc_datetime; +use sqlbuild_python_planner::freshness::main::row_record::row_record; use sqlbuild_python_project::compiled_project::main::plain_value::plain_value; use sqlbuild_python_project::fingerprints::main::row_fingerprint::row_fingerprint; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/changed.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/changed.rs index aafe27a81f..27c939e00c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/changed.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/changed.rs @@ -6,8 +6,10 @@ use sqlbuild_core::cli_text::main::format_tree_connector::format_tree_connector; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_state::consumption::models::{HeldBack, NodeKey, RunReason}; -use crate::bindings::_helpers::consumption::policy::{kind_value, reach_value}; -use crate::bindings::_helpers::consumption::recorder::{ChangedDecision, run_reason_text}; +use sqlbuild_python_planner::consumption::main::kind_value::kind_value; +use sqlbuild_python_planner::consumption::main::reach_value::reach_value; +use sqlbuild_python_planner::consumption::main::run_reason_text::run_reason_text; +use sqlbuild_python_planner::consumption::models::ChangedDecision; fn node(key: &NodeKey) -> String { format!("{}:{}", key.kind, key.name) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 2ffe94163e..b5b8167d49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -20,12 +20,8 @@ use crate::bindings::_helpers::command_runtime::preview_target::validate_preview use crate::bindings::_helpers::command_runtime::runtime_errors::raised; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; -use crate::bindings::_helpers::consumption::policy::{changed_policy, policy_line}; -use crate::bindings::_helpers::consumption::recorder::{ChangedDecision, preview_changed}; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::plan_command::changed::{changed_json, changed_text}; -use crate::bindings::_helpers::plan_output::plan_json::{CHANGED_KEY, format_plan_json}; -use crate::bindings::_helpers::plan_output::plan_text::format_plan; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -37,6 +33,13 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_front::compile_attachment::main::build_effective_settings_config::build_effective_settings_config; +use sqlbuild_python_planner::consumption::main::changed_policy::changed_policy; +use sqlbuild_python_planner::consumption::main::policy_line::policy_line; +use sqlbuild_python_planner::consumption::main::preview_changed::preview_changed; +use sqlbuild_python_planner::consumption::models::ChangedDecision; +use sqlbuild_python_planner::plan_output::constants::CHANGED_KEY; +use sqlbuild_python_planner::plan_output::main::format_plan::format_plan; +use sqlbuild_python_planner::plan_output::main::format_plan_json::format_plan_json; const COMMAND_NAME: &str = "plan"; const DEFAULT_SECTION_ENTRIES: i64 = 50; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs index a4063abaf5..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs @@ -1,9 +1 @@ -pub(crate) mod cursor_details; -pub(crate) mod display; -pub(crate) mod migrations; -pub(crate) mod plan_json; -pub(crate) mod plan_text; -pub(crate) mod python_identity; pub(crate) mod registration; -pub(crate) mod safety; -pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs index 96c0593e33..285db800b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs @@ -2,12 +2,16 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::plan_output::{display, migrations, plan_json, plan_text, safety}; +use sqlbuild_python_planner::plan_output::main::register_display::register_display; +use sqlbuild_python_planner::plan_output::main::register_migrations::register_migrations; +use sqlbuild_python_planner::plan_output::main::register_plan_json::register_plan_json; +use sqlbuild_python_planner::plan_output::main::register_plan_text::register_plan_text; +use sqlbuild_python_planner::plan_output::main::register_safety::register_safety; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - display::register(module)?; - migrations::register(module)?; - plan_json::register(module)?; - plan_text::register(module)?; - safety::register(module) + register_display(module)?; + register_migrations(module)?; + register_plan_json(module)?; + register_plan_text(module)?; + register_safety(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/mod.rs index 301a6f231c..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/mod.rs @@ -1,6 +1 @@ -pub(crate) mod actions; -pub(crate) mod detect; -pub(crate) mod metadata; pub(crate) mod registration; -pub(crate) mod renames; -pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/registration.rs index 750b787720..84853c8e28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/registration.rs @@ -2,11 +2,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::planner_changes::{actions, detect, metadata, renames}; +use sqlbuild_python_planner::planner_changes::main::register_actions::register_actions; +use sqlbuild_python_planner::planner_changes::main::register_detect::register_detect; +use sqlbuild_python_planner::planner_changes::main::register_metadata::register_metadata; +use sqlbuild_python_planner::planner_changes::main::register_renames::register_renames; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - actions::register(module)?; - detect::register(module)?; - metadata::register(module)?; - renames::register(module) + register_actions(module)?; + register_detect(module)?; + register_metadata(module)?; + register_renames(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs index 584c84460b..dc54b70121 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -1,15 +1,3 @@ //! Plan entry and plan output assembly over native planner resolution. -pub(crate) mod clone_plans; -pub(crate) mod command_plans; -pub(crate) mod contexts; -pub(crate) mod display_plan; -pub(crate) mod execution_plan; -pub(crate) mod getters; -pub(crate) mod model_entries; -pub(crate) mod models; -pub(crate) mod plan_output; pub(crate) mod registration; -pub(crate) mod relations; -mod test_planning; -mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs deleted file mode 100644 index e29ce386a0..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs +++ /dev/null @@ -1,99 +0,0 @@ -//! Typed requests of plan entry construction. - -use pyo3::prelude::{Bound, Py, PyAny}; -use pyo3::pyclass; -use sqlbuild_python_project::native_host::models::ModelLookup; - -/// What resolving relation locations and source metadata for one scope reads. -#[derive(Debug)] -pub(crate) struct RelationsRequest<'py> { - pub(crate) project: Bound<'py, PyAny>, - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) connection: Bound<'py, PyAny>, - pub(crate) scope: Bound<'py, PyAny>, - pub(crate) deferral: Bound<'py, PyAny>, - pub(crate) project_config: Bound<'py, PyAny>, - pub(crate) local_config: Bound<'py, PyAny>, - pub(crate) known_source_columns: Bound<'py, PyAny>, - pub(crate) relation_keys: Bound<'py, PyAny>, -} - -/// What building model plan entries for one execution scope reads. -#[derive(Debug)] -pub(crate) struct PlanEntriesRequest<'py> { - pub(crate) project: Bound<'py, PyAny>, - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) scope: Bound<'py, PyAny>, - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) relations: Bound<'py, PyAny>, - pub(crate) resolved_models: Bound<'py, PyAny>, - pub(crate) cursor_overrides: Bound<'py, PyAny>, - pub(crate) full_refresh: bool, - pub(crate) build_inputs: Bound<'py, PyAny>, - /// The effective worker limit execution will use. - pub(crate) concurrency: i64, -} - -/// What assembling the plan output's fields reads. -#[derive(Debug)] -pub(crate) struct PlanOutputInputs<'py> { - pub(crate) project: Bound<'py, PyAny>, - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) scope: Bound<'py, PyAny>, - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) relations: Bound<'py, PyAny>, - pub(crate) changes: Bound<'py, PyAny>, - pub(crate) model_entries: Vec>, - pub(crate) model_warnings: Vec>, - pub(crate) reload_sources: Bound<'py, PyAny>, - pub(crate) extras: Bound<'py, PyAny>, -} - -/// Resolved relation and source inputs for plan entry construction. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct PlannerRelationsContext { - pub(crate) model_locations: Py, - pub(crate) seed_locations: Py, - pub(crate) function_locations: Py, - pub(crate) source_map: Py, - pub(crate) source_read_map: Py, - pub(crate) python_source_read_map: Py, - pub(crate) source_warehouse_columns: Py, - pub(crate) star_exclude_keyword: Py, - pub(crate) listed_source_names: Py, -} - -/// Relation lookups and source columns for building one model plan entry. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct ModelPlanContext { - pub(crate) model_locations: Py, - pub(crate) models_by_name: ModelLookup, - pub(crate) seed_locations: Py, - pub(crate) function_locations: Py, - pub(crate) source_map: Py, - pub(crate) source_warehouse_columns: Py, - pub(crate) star_exclude_keyword: Py, - pub(crate) functions_by_name: Py, - pub(crate) runtime_cursor_producer_names: Py, - pub(crate) future_cursor_config: Py, - pub(crate) start_cursor_config: Py, - pub(crate) invocation_time: Py, -} - -/// Blocked models and cursor overrides for plan entry building. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct PlanEntryBuildInputs { - pub(crate) source_freshness_blocked_model_names: Py, - pub(crate) external_blocked_model_names: Py, - pub(crate) start_cursor_override: Py, - pub(crate) end_cursor_override: Py, - pub(crate) future_cursor_config: Py, - pub(crate) start_cursor_config: Py, - pub(crate) invocation_time: Py, - pub(crate) max_microbatches: Py, - pub(crate) max_microbatches_is_override: bool, - pub(crate) microbatch_limit_action: Py, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs index ed45e8827f..174bc7b89f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs @@ -1,17 +1,26 @@ //! Register plan entry and plan output assembly entry points. use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use sqlbuild_python_planner::planner_entries::main::entries::register_model_entries::register_model_entries; +use sqlbuild_python_planner::planner_entries::main::entries::register_relations::register_relations; +use sqlbuild_python_planner::planner_entries::main::plans::register_clone_plans::register_clone_plans; +use sqlbuild_python_planner::planner_entries::main::plans::register_command_plans::register_command_plans; +use sqlbuild_python_planner::planner_entries::main::plans::register_display_plan::register_display_plan; +use sqlbuild_python_planner::planner_entries::main::plans::register_execution_plan::register_execution_plan; +use sqlbuild_python_planner::planner_entries::main::plans::register_plan_output::register_plan_output; +use sqlbuild_python_planner::planner_entries::main::plans::register_test_planning::register_test_planning; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - super::clone_plans::register(module)?; - super::command_plans::register(module)?; - super::display_plan::register(module)?; - super::execution_plan::register(module)?; - super::model_entries::register(module)?; - super::plan_output::register(module)?; - super::relations::register(module)?; - super::test_planning::register(module) + module + .add_class::()?; + module.add_class::()?; + module.add_class::()?; + register_clone_plans(module)?; + register_command_plans(module)?; + register_display_plan(module)?; + register_execution_plan(module)?; + register_model_entries(module)?; + register_plan_output(module)?; + register_relations(module)?; + register_test_planning(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs index f8f6364a35..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs @@ -1,4 +1 @@ -pub(crate) mod identities; -pub(crate) mod metadata; pub(crate) mod registration; -pub(crate) mod staleness; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs index cce0b3f494..4dd3a32767 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs @@ -2,14 +2,16 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::planner_identity::{identities, metadata, staleness}; +use sqlbuild_python_planner::planner_identity::main::register_identities::register_identities; +use sqlbuild_python_planner::planner_identity::main::register_metadata::register_metadata; +use sqlbuild_python_planner::planner_identity::main::register_staleness::register_staleness; use sqlbuild_python_project::planner_identity::main::register_graph::register_graph; use sqlbuild_python_project::planner_identity::main::register_hashing::register_hashing; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { register_graph(module)?; register_hashing(module)?; - identities::register(module)?; - metadata::register(module)?; - staleness::register(module) + register_identities(module)?; + register_metadata(module)?; + register_staleness(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs deleted file mode 100644 index 09e5891b19..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs +++ /dev/null @@ -1,14 +0,0 @@ -use sqlbuild_host::entries::models::AdapterHost; -pub(super) const AUTOMATIC: &str = "automatic"; -pub(super) const DONE: &str = "done"; -pub(super) const PENDING_ARCHIVE: &str = "pending_archive"; -pub(super) const PENDING_VIEW: &str = "pending_view"; -pub(super) const LIST_RELATIONS_FOR_INSPECTION: AdapterHost = - AdapterHost::ListRelationsForInspection; -pub(super) const GET_COLUMNS_FOR_INSPECTION: AdapterHost = AdapterHost::GetColumnsForInspection; -pub(super) const QUALIFIED_RELATION_MAX_PARTS: usize = 3; -pub(super) const MIGRATION_MODEL_NAME_METADATA_KEY: &str = "model_name"; -pub(super) const HIDDEN_ORIGIN_REMEDY: &str = "the connection may lack permission to see it, or it was dropped outside SQLBuild; restore access to it, or remove migrate_from to build the model fresh deliberately (missing_migration_origin does not apply)"; -pub(super) const MISSING_ORIGIN_ALLOW_OUTCOME: &str = "nothing is migrated and the build continues as if migrate_from were absent; remove migrate_from once no target still needs it"; -pub(super) const MISSING_ORIGIN_CONFIRMATION_OUTCOME: &str = "building without it needs confirmation (missing_migration_origin = require_confirmation); pass --allow-missing-migration-origin, or remove migrate_from if it is no longer needed"; -pub(super) const MISSING_ORIGIN_DENY_OUTCOME: &str = "the build stops (missing_migration_origin = deny); if the migration already happened elsewhere or is no longer needed, remove migrate_from"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index b5361f630c..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -1,35 +1 @@ -mod column_candidates; -mod column_entries; -mod column_objects; -mod column_overlay; -pub(crate) mod column_planning; -mod column_warnings; -pub(crate) mod columns; -mod compatibility; -mod constants; -mod discovery; -mod discovery_events; -mod discovery_models; -mod discovery_requests; -mod errors; -pub(crate) mod fingerprint; -pub(crate) mod getters; -mod known_input_columns; -mod matching; -mod model_entries; -mod model_handover; -mod model_overlay; -pub(crate) mod model_planning; -mod model_warnings; -pub(crate) mod models; -mod old_name_entries; -mod old_name_objects; -pub(crate) mod old_name_planning; -mod old_name_references; -pub(crate) mod old_names; -mod planning_boundary; -mod planning_execution; -mod planning_origin; -pub(crate) mod projections; pub(crate) mod registration; -pub(crate) mod state_inspection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs deleted file mode 100644 index 4d04495439..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs +++ /dev/null @@ -1,150 +0,0 @@ -use pyo3::prelude::{Bound, PyAny}; -use pyo3::types::PyDict; - -#[derive(Debug)] -pub(crate) struct OldNamePlanner<'py> { - pub runtime: Bound<'py, PyAny>, - pub models: Vec>, - pub histories: Vec>, - pub views: Bound<'py, PyAny>, - pub columns: Bound<'py, PyAny>, - pub retention: Bound<'py, PyAny>, - pub now: Bound<'py, PyAny>, -} - -#[derive(Debug)] -pub(crate) struct ColumnCandidate<'py> { - pub model: Bound<'py, PyAny>, - pub warehouse: Bound<'py, PyDict>, - pub declared: Bound<'py, PyDict>, - pub fingerprint: Bound<'py, PyAny>, - pub previous: Bound<'py, PyAny>, - pub current: Bound<'py, PyAny>, -} -#[derive(Debug)] -pub(crate) struct CheckedColumns<'py> { - pub entries: Vec>, - pub warnings: Vec>, -} - -/// The planning-phase inputs every migration planner reads. -#[derive(Debug)] -pub(crate) struct MigrationPhase<'py> { - pub(crate) runtime: Bound<'py, PyAny>, - pub(crate) scope: Bound<'py, PyAny>, - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) overrides: Bound<'py, PyAny>, - pub(crate) deferral: Bound<'py, PyAny>, -} - -/// The column migration planning inputs beyond the phase. -#[derive(Debug)] -pub(crate) struct ColumnMigrationInputs<'py> { - pub(crate) full_refresh_model_names: Bound<'py, PyAny>, - pub(crate) physical_relations: Bound<'py, PyAny>, - pub(crate) source_columns: Bound<'py, PyAny>, -} - -/// The snapshot after model migrations overlay it, with the planned entries and their warnings. -#[derive(Debug)] -pub(crate) struct ModelMigrationPlanning<'py> { - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) entries: Vec>, - pub(crate) warnings: Vec>, -} - -/// The snapshot after column renames overlay it, with the planned entries, rename hints and warnings. -#[derive(Debug)] -pub(crate) struct ColumnMigrationPlanning<'py> { - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) entries: Vec>, - pub(crate) hints: Vec>, - pub(crate) warnings: Vec>, -} - -/// The old-name view steps, the recorded live views and the warnings they raise. -#[derive(Debug)] -pub(crate) struct OldNameViewPlanning<'py> { - pub(crate) entries: Bound<'py, PyAny>, - pub(crate) views: Bound<'py, PyAny>, - pub(crate) warnings: Vec>, -} - -/// One destination model whose data should come from an earlier relation. -#[derive(Debug, Clone)] -pub(crate) struct ModelMigrationRequest<'py> { - pub(crate) model: Bound<'py, PyAny>, - pub(crate) discovery: Bound<'py, PyAny>, - pub(crate) raw_origin: Option>, - pub(crate) origin_location: Option>, - pub(crate) origin_model: Bound<'py, PyAny>, - pub(crate) force: bool, - pub(crate) identity_only: bool, -} - -/// One project model's explicit migrate_from, resolved to a relation. -#[derive(Debug)] -pub(crate) struct ModelMigrationDeclaration<'py> { - pub(crate) model_name: Bound<'py, PyAny>, - pub(crate) origin_location: Bound<'py, PyAny>, - pub(crate) origin_model: Bound<'py, PyAny>, -} - -/// Manual and automatically discovered migration requests for one plan. -#[derive(Debug)] -pub(crate) struct ModelMigrationDiscovery<'py> { - pub(crate) requests: Vec>, - pub(crate) warnings: Vec>, - pub(crate) destination_fingerprints: Bound<'py, PyDict>, -} - -/// Compatibility status plus human-readable blocking findings. -#[derive(Debug)] -pub(crate) struct MigrationCompatibilityResult<'py> { - pub(crate) status: Bound<'py, PyAny>, - pub(crate) findings: Bound<'py, PyAny>, -} - -/// The fingerprint inputs shared by migration matching and destination fingerprints. -#[derive(Debug)] -pub(crate) struct FingerprintInputs<'py> { - pub(crate) metadata_jsons: Bound<'py, PyAny>, - pub(crate) dialect: Bound<'py, PyAny>, - pub(crate) fingerprints: Bound<'py, PyAny>, -} - -/// One resolved migration request decided against live state. -#[derive(Debug)] -pub(crate) struct MigrationDecisionRequest<'a, 'py> { - pub(crate) runtime: Bound<'py, PyAny>, - pub(crate) request: &'a ModelMigrationRequest<'py>, - pub(crate) origin: Bound<'py, PyAny>, - pub(crate) origin_model: Bound<'py, PyAny>, - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) state: Bound<'py, PyAny>, - pub(crate) snapshots_config: Bound<'py, PyAny>, -} - -/// What checking an origin relation's compatibility with its destination model reads. -#[derive(Debug)] -pub(crate) struct CompatibilityCheck<'py> { - pub(crate) model: Bound<'py, PyAny>, - pub(crate) origin_relation: Bound<'py, PyAny>, - pub(crate) origin_columns: Bound<'py, PyAny>, - pub(crate) sql_analysis_enabled: Bound<'py, PyAny>, - pub(crate) column_dialect: Bound<'py, PyAny>, - pub(crate) snapshots_config: Bound<'py, PyAny>, -} - -/// What discovering automatic and manual migration requests reads. -#[derive(Debug)] -pub(crate) struct DiscoveryRequest<'py> { - pub(crate) runtime: Bound<'py, PyAny>, - pub(crate) scope: Bound<'py, PyAny>, - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) manual_requests: Vec>, - pub(crate) declarations: Vec>, - pub(crate) state: Bound<'py, PyAny>, - pub(crate) project_schemas: Bound<'py, PyAny>, - pub(crate) fingerprints: Bound<'py, PyAny>, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs index 8e819c060c..1d370fd5a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs @@ -1,11 +1,17 @@ //! Register pure migration helpers. use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use sqlbuild_python_planner::planner_migrations::main::register_column_planning::register_column_planning; +use sqlbuild_python_planner::planner_migrations::main::register_columns::register_columns; +use sqlbuild_python_planner::planner_migrations::main::register_fingerprint::register_fingerprint; +use sqlbuild_python_planner::planner_migrations::main::register_projections::register_projections; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - super::columns::register(module)?; - super::column_planning::register(module)?; - super::projections::register(module)?; - module.add_class::()?; - super::fingerprint::register(module) + register_columns(module)?; + register_column_planning(module)?; + register_projections(module)?; + module + .add_class::( + )?; + register_fingerprint(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs deleted file mode 100644 index 5c45d15791..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! Python modules and names the planner phases exchange values with. - -pub(crate) const SNOWFLAKE_ADAPTER: &str = "snowflake"; -pub(crate) const BIGQUERY_ADAPTER: &str = "bigquery"; -pub(crate) const TRANSIENT_TABLE_TYPE: &str = "transient"; -pub(crate) const PERMANENT_TABLE_TYPE: &str = "permanent"; -pub(crate) const DECREASE_WARNING: &str = - "Retention decreases are irreversible once warehouse history expires."; -pub(crate) const DOWNGRADE_HISTORY_DAYS: i64 = 90; -pub(crate) const FULL_REFRESH_DISABLED_REBUILD_CODE: &str = "S203"; -pub(crate) const MISSING_UPSTREAM_CODE: &str = "S301"; -pub(crate) const RETENTION_DECREASE: &str = "decrease"; -pub(crate) const RETENTION_INCREASE: &str = "increase"; -pub(crate) const RETENTION_MATCH: &str = "match"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs index b095876f5b..b84ad98904 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs @@ -1,16 +1,3 @@ //! Native execution-planning phases over Python planner models and adapter callbacks. -pub(crate) mod buildability; -pub(crate) mod constants; -pub(crate) mod contexts; -pub(crate) mod deferred_origins; -pub(crate) mod execution_changes; -pub(crate) mod full_refresh; -pub(crate) mod getters; -pub(crate) mod identities; -pub(crate) mod models; -pub(crate) mod output_assembly; pub(crate) mod registration; -pub(crate) mod retention; -pub(crate) mod runtime; -pub(crate) mod warehouse_state; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs deleted file mode 100644 index c1479d9f2b..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs +++ /dev/null @@ -1,107 +0,0 @@ -//! Plan output fields gathered across the plan phases before the output is built once. - -use std::collections::BTreeMap; - -use pyo3::prelude::{Bound, Py, PyAny}; -use pyo3::pyclass; - -/// The plan output's fields and metadata entries, each set before the output is built. -#[derive(Debug)] -pub(crate) struct PlanOutputFields<'py> { - pub(crate) fields: Vec<(&'static str, Bound<'py, PyAny>)>, - pub(crate) metadata: Vec<(&'static str, Bound<'py, PyAny>)>, -} - -/// The model changes and resolved actions after write-hash reconciliation. -#[derive(Debug)] -pub(crate) struct ChangeReconciliation<'py> { - pub(crate) changes: Bound<'py, PyAny>, - pub(crate) resolved_actions: Bound<'py, PyAny>, -} - -/// Planned model entries and the warnings planning them raised. -#[derive(Debug)] -pub(crate) struct ModelEntryResults<'py> { - pub(crate) entries: Vec>, - pub(crate) warnings: Vec>, -} - -/// The runtime, warehouse state, identities and overrides the base plan output reads. -#[derive(Debug)] -pub(crate) struct PlanOutputContext<'a, 'py> { - pub(crate) runtime: &'a Bound<'py, PyAny>, - pub(crate) warehouse: &'a Bound<'py, PyAny>, - pub(crate) identities: &'a Bound<'py, PyAny>, - pub(crate) overrides: &'a Bound<'py, PyAny>, -} - -/// The execution scope, reconciled changes, planned entries and source freshness of a plan. -#[derive(Debug)] -pub(crate) struct PlannedModelEntries<'a, 'py> { - pub(crate) execution_scope: &'a Bound<'py, PyAny>, - pub(crate) reconciliation: &'a ChangeReconciliation<'py>, - pub(crate) entries: ModelEntryResults<'py>, - pub(crate) source_freshness: &'a Bound<'py, PyAny>, -} - -/// The runtime, warehouse state, scopes and policies storage policy planning reads. -#[derive(Debug)] -pub(crate) struct StoragePolicyContext<'a, 'py> { - pub(crate) runtime: &'a Bound<'py, PyAny>, - pub(crate) warehouse: &'a Bound<'py, PyAny>, - pub(crate) scopes: &'a Bound<'py, PyAny>, - pub(crate) policies: &'a Bound<'py, PyAny>, -} - -/// The scopes, warehouse state, identities and changes stale-selection warnings read. -#[derive(Debug)] -pub(crate) struct PlanWarningContext<'a, 'py> { - pub(crate) scopes: &'a Bound<'py, PyAny>, - pub(crate) warehouse: &'a Bound<'py, PyAny>, - pub(crate) identities: &'a Bound<'py, PyAny>, - pub(crate) stale_warning_changes: &'a Bound<'py, PyAny>, -} - -/// Resolved execution environment shared by planner phases. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct PlannerRuntime { - pub(crate) project: Py, - pub(crate) adapter: Py, - pub(crate) connection: Py, - pub(crate) project_config: Py, - pub(crate) local_config: Py, - pub(crate) on_progress: Py, - pub(crate) invocation_time: Py, -} - -/// Warehouse snapshot, inspection relations and migration plans gathered once per plan. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct PlannerWarehouseState { - pub(crate) snapshot: Py, - pub(crate) inspection_relations: Py, - pub(crate) migration_entries: Py, - pub(crate) migration_warnings: Py, - pub(crate) column_migration_entries: Py, - pub(crate) column_rename_hints: Py, - pub(crate) old_name_view_entries: Py, - pub(crate) old_name_views: Py, - pub(crate) migration_fingerprints: Py, -} - -/// Expected version identities for inspection and stale-warning scopes. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct PlannerIdentityContext { - pub(crate) version_identities: Py, - pub(crate) stale_warning_identities: Py, -} - -/// Planner identity work since the last reset, for work-count tests. -#[pyclass] -#[derive(Debug, Default)] -pub(crate) struct IdentityWork { - pub(crate) identity_builds: u64, - pub(crate) stale_detections: BTreeMap, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs index 5a1d86ec23..6f51aa9d97 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -1,44 +1,28 @@ //! Register the native execution-planning phases. use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::wrap_pyfunction; use sqlbuild_python_project::planner_scope::main::resolve_planner_scopes_function::resolve_planner_scopes_function; +use sqlbuild_python_planner::planner_phases::main::outputs::detect_stale_warning_changes_of_function::detect_stale_warning_changes_of_function; +use sqlbuild_python_planner::planner_phases::main::outputs::planner_identity_work_info_function::planner_identity_work_info_function; +use sqlbuild_python_planner::planner_phases::main::rebuilds::check_full_rebuild_allowed_function::check_full_rebuild_allowed_function; +use sqlbuild_python_planner::planner_phases::main::rebuilds::describe_full_rebuild_cause_function::describe_full_rebuild_cause_function; +use sqlbuild_python_planner::planner_phases::main::rebuilds::plan_retention_function::plan_retention_function; +use sqlbuild_python_planner::planner_phases::main::rebuilds::plan_table_types_function::plan_table_types_function; +use sqlbuild_python_planner::planner_phases::main::rebuilds::table_type_copy_name_function::table_type_copy_name_function; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module - .add_class::()?; + module.add_class::()?; + module.add_class::()?; module.add_class::()?; module - .add_class::()?; + .add_class::()?; module.add_function(resolve_planner_scopes_function(module)?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::identities::planner_identity_work_info, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::identities::detect_stale_warning_changes_of, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::full_refresh::describe_full_rebuild_cause, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::full_refresh::check_full_rebuild_allowed, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::retention::table_type_copy_name, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::retention::plan_table_types, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::retention::plan_retention, - module - )?)?; + module.add_function(planner_identity_work_info_function(module)?)?; + module.add_function(detect_stale_warning_changes_of_function(module)?)?; + module.add_function(describe_full_rebuild_cause_function(module)?)?; + module.add_function(check_full_rebuild_allowed_function(module)?)?; + module.add_function(table_type_copy_name_function(module)?)?; + module.add_function(plan_table_types_function(module)?)?; + module.add_function(plan_retention_function(module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/constants.rs deleted file mode 100644 index b443c6acb1..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/constants.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! Planner cursor configuration keys. - -pub(super) const CURSOR_START: &str = "cursor_start"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs index 93bab4d84b..a21268c7ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs @@ -1,20 +1,3 @@ //! Planner reference resolution with Python adapter rendering callbacks. -mod constants; -pub(crate) mod cursor_bounds; -pub(crate) mod cursor_model; -pub(crate) mod cursor_overrides; -pub(crate) mod cursor_policies; -pub(crate) mod cursor_ranges; -mod cursor_safety; -pub(crate) mod cursor_warnings; -pub(crate) mod cursor_windows; -pub(crate) mod future_safety; -pub(crate) mod locations; -pub(crate) mod maximum_safety; -pub(crate) mod models; -pub(crate) mod orchestration; -pub(crate) mod refs; pub(crate) mod registration; -pub(crate) mod sources; -mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs deleted file mode 100644 index db9d2e26f5..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! Typed requests of planner SQL resolution. - -use pyo3::prelude::{Bound, PyAny}; - -/// The cursor window that filters the cursor inputs one resolution reads. -#[derive(Debug)] -pub(crate) struct CursorFiltering<'py> { - pub(crate) bounds: Bound<'py, PyAny>, - pub(crate) filter_inputs: Bound<'py, PyAny>, - pub(crate) cursor_type: Bound<'py, PyAny>, - pub(crate) lower_bound_inclusive: bool, -} - -/// One model's SQL resolution decisions. -#[derive(Debug)] -pub(crate) struct ModelSqlRequest<'py> { - pub(crate) model: Bound<'py, PyAny>, - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) backfill: Bound<'py, PyAny>, - pub(crate) full_refresh: bool, - pub(crate) cursor_overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), - pub(crate) replaces_relation: bool, - pub(crate) full_rebuild_cause: Option, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs index 86ba08ef70..db1babf5ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs @@ -1,17 +1,28 @@ //! Register planner SQL resolution entry points. use pyo3::prelude::{Bound, PyModule, PyResult}; +use sqlbuild_python_planner::planner_resolution::main::cursor::register_cursor_bounds::register_cursor_bounds; +use sqlbuild_python_planner::planner_resolution::main::cursor::register_cursor_overrides::register_cursor_overrides; +use sqlbuild_python_planner::planner_resolution::main::cursor::register_cursor_ranges::register_cursor_ranges; +use sqlbuild_python_planner::planner_resolution::main::cursor::register_cursor_warnings::register_cursor_warnings; +use sqlbuild_python_planner::planner_resolution::main::cursor::register_cursor_windows::register_cursor_windows; +use sqlbuild_python_planner::planner_resolution::main::references::register_locations::register_locations; +use sqlbuild_python_planner::planner_resolution::main::references::register_orchestration::register_orchestration; +use sqlbuild_python_planner::planner_resolution::main::references::register_refs::register_refs; +use sqlbuild_python_planner::planner_resolution::main::references::register_sources::register_sources; +use sqlbuild_python_planner::planner_resolution::main::safety::register_future_safety::register_future_safety; +use sqlbuild_python_planner::planner_resolution::main::safety::register_maximum_safety::register_maximum_safety; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - super::cursor_bounds::register(module)?; - super::cursor_overrides::register(module)?; - super::cursor_ranges::register(module)?; - super::cursor_warnings::register(module)?; - super::future_safety::register(module)?; - super::maximum_safety::register(module)?; - super::cursor_windows::register(module)?; - super::refs::register(module)?; - super::locations::register(module)?; - super::orchestration::register(module)?; - super::sources::register(module) + register_cursor_bounds(module)?; + register_cursor_overrides(module)?; + register_cursor_ranges(module)?; + register_cursor_warnings(module)?; + register_future_safety(module)?; + register_maximum_safety(module)?; + register_cursor_windows(module)?; + register_refs(module)?; + register_locations(module)?; + register_orchestration(module)?; + register_sources(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs deleted file mode 100644 index 230067769f..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! Relation categories with additional scenario binding semantics. - -pub(super) const REF: &str = "ref"; -pub(super) const SOURCE: &str = "source"; -pub(super) const SOURCE_MARKER: usize = 2; -pub(super) const DBT_REF: &str = "dbt_ref"; -pub(super) const SCENARIO_PLAN_INVALID_HASH_PREFIX: &str = "S501"; -pub(super) const SCENARIO_PLAN_HASH_COLLISION: &str = "S502"; -pub(super) const SCENARIO_PLAN_RELATION_COLLISION: &str = "S503"; -pub(super) const SCENARIO_PLAN_GRAPH_VALIDATION: &str = "S504"; -pub(super) const SCENARIO_PLAN_UNKNOWN_SEED: &str = "S507"; -pub(super) const SCENARIO_PLAN_MISSING_FIXTURE_SQL: &str = "S508"; -pub(super) const SCENARIO_PLAN_MISSING_RELATION_TARGET: &str = "S509"; -pub(super) const SCENARIO_PLAN_INVALID_FIXTURE: &str = "S510"; -pub(super) const SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER: &str = "S511"; -pub(super) const SCENARIO_PLAN_EXPECTED_COLUMNS: &str = "S512"; -pub(super) const SCENARIO_EXPECTED_COLUMNS_HELP: &str = "list every column the model outputs in the expected CTE, and remove columns it does not output"; -pub(super) const SCENARIO_PLAN_INTERNAL: &str = "S599"; -/// The zero-argument cursor intrinsics a model query can call. -pub(super) const CURSOR_INTRINSIC_NAMES: [&str; 2] = ["__cursor_start", "__cursor_end"]; -pub(super) const INCREMENTAL_MODE_KEY: &str = "incremental_mode"; -pub(super) const MICROBATCH_INCREMENTAL_MODE: &str = "microbatch"; -pub(super) const CURSOR_TYPE_KEY: &str = "cursor_type"; -pub(super) const INTEGER_CURSOR_TYPE: &str = "integer"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs index 3bacaebca1..43c07cb865 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs @@ -1,15 +1,3 @@ //! Python boundaries for native scenario planning. -pub(crate) mod artifacts; -mod cli; -mod constants; -mod entries; -pub(crate) mod execution; -mod fixtures; -pub(crate) mod graph; -mod helper_scope; -mod models; pub(crate) mod registration; -mod relations; -mod sql; -mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs deleted file mode 100644 index 70ff75b823..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! Borrowed Python callback boundaries for scenario SQL assembly. - -use pyo3::prelude::{Bound, PyAny}; - -#[derive(Debug)] -pub(crate) struct SqlContext<'py> { - pub adapter: Bound<'py, PyAny>, - pub relations: Bound<'py, PyAny>, - pub syntax: Bound<'py, PyAny>, - pub label: String, -} - -/// One scenario's graph and the relation map and qualification that locate its relations. -#[derive(Debug)] -pub(crate) struct ScenarioRelations<'py> { - pub(crate) project: Bound<'py, PyAny>, - pub(crate) graph_plan: Bound<'py, PyAny>, - pub(crate) relation_map: Bound<'py, PyAny>, - pub(crate) render_qualified_name: Bound<'py, PyAny>, - pub(crate) database: Bound<'py, PyAny>, - pub(crate) schema: Bound<'py, PyAny>, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs index c2e450af9b..574dcf8e25 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs @@ -1,50 +1,31 @@ //! Register scenario planning bindings. -use crate::bindings::_helpers::planner_scenario::{ - artifacts, cli, execution, fixtures, graph, relations, sql, -}; +use sqlbuild_python_planner::planner_scenario::main::build_scenario_artifact_name_function::build_scenario_artifact_name_function; +use sqlbuild_python_planner::planner_scenario::main::build_scenario_hash_index_function::build_scenario_hash_index_function; +use sqlbuild_python_planner::planner_scenario::main::build_scenario_relation_map_function::build_scenario_relation_map_function; +use sqlbuild_python_planner::planner_scenario::main::compute_scenario_hash_prefix_function::compute_scenario_hash_prefix_function; +use sqlbuild_python_planner::planner_scenario::main::scenario_artifact_physical_name_check_function::scenario_artifact_physical_name_check_function; +use sqlbuild_python_planner::planner_scenario::main::parse_scenario_artifact_physical_name_function::parse_scenario_artifact_physical_name_function; +use sqlbuild_python_planner::planner_scenario::main::build_cli_scenario_plan_function::build_cli_scenario_plan_function; +use sqlbuild_python_planner::planner_scenario::main::build_scenario_execution_plan_function::build_scenario_execution_plan_function; +use sqlbuild_python_planner::planner_scenario::main::build_scenario_fixture_plans_function::build_scenario_fixture_plans_function; +use sqlbuild_python_planner::planner_scenario::main::plan_scenario_graph_function::plan_scenario_graph_function; +use sqlbuild_python_planner::planner_scenario::main::build_scenario_relation_plan_of_function::build_scenario_relation_plan_of_function; +use sqlbuild_python_planner::planner_scenario::main::resolve_scenario_check_sql_function::resolve_scenario_check_sql_function; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::wrap_pyfunction; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - execution::build_scenario_execution_plan, - module - )?)?; - module.add_function(wrap_pyfunction!( - fixtures::build_scenario_fixture_plans, - module - )?)?; - module.add_function(wrap_pyfunction!(sql::resolve_scenario_check_sql, module)?)?; - module.add_function(wrap_pyfunction!( - artifacts::compute_scenario_hash_prefix, - module - )?)?; - module.add_function(wrap_pyfunction!( - artifacts::build_scenario_hash_index, - module - )?)?; - module.add_function(wrap_pyfunction!( - artifacts::build_scenario_artifact_name, - module - )?)?; - module.add_function(wrap_pyfunction!( - artifacts::parse_scenario_artifact_physical_name, - module - )?)?; - module.add_function(wrap_pyfunction!( - artifacts::is_scenario_artifact_physical_name, - module - )?)?; - module.add_function(wrap_pyfunction!( - artifacts::build_scenario_relation_map, - module - )?)?; - module.add_function(wrap_pyfunction!(graph::plan_scenario_graph, module)?)?; - module.add_function(wrap_pyfunction!( - relations::build_scenario_relation_plan_of, - module - )?)?; - module.add_function(wrap_pyfunction!(cli::build_cli_scenario_plan, module)?)?; + module.add_function(build_scenario_execution_plan_function(module)?)?; + module.add_function(build_scenario_fixture_plans_function(module)?)?; + module.add_function(resolve_scenario_check_sql_function(module)?)?; + module.add_function(compute_scenario_hash_prefix_function(module)?)?; + module.add_function(build_scenario_hash_index_function(module)?)?; + module.add_function(build_scenario_artifact_name_function(module)?)?; + module.add_function(parse_scenario_artifact_physical_name_function(module)?)?; + module.add_function(scenario_artifact_physical_name_check_function(module)?)?; + module.add_function(build_scenario_relation_map_function(module)?)?; + module.add_function(plan_scenario_graph_function(module)?)?; + module.add_function(build_scenario_relation_plan_of_function(module)?)?; + module.add_function(build_cli_scenario_plan_function(module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs index 8176115560..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs @@ -1,11 +1 @@ -pub(crate) mod audit_entries; -pub(crate) mod audits; -pub(crate) mod cursor_inputs; -pub(crate) mod cursor_types; -pub(crate) mod lineage; -pub(crate) mod microbatch_limits; -pub(crate) mod microbatches; -pub(crate) mod model_entries; pub(crate) mod registration; -pub(crate) mod strategy; -pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index 5ae6a39806..4fed3db332 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -1,47 +1,36 @@ //! Register the planner strategy. +use pyo3::prelude::PyModuleMethods; use pyo3::prelude::{Bound, PyModule, PyResult}; -use pyo3::{prelude::PyModuleMethods, wrap_pyfunction}; -use crate::bindings::_helpers::planner_strategy::{audits, cursor_types, microbatches, strategy}; +use sqlbuild_python_planner::planner_strategy::main::audits::register_audits::register_audits; +use sqlbuild_python_planner::planner_strategy::main::cursor_types::register_cursor_types::register_cursor_types; +use sqlbuild_python_planner::planner_strategy::main::microbatches::register_microbatches::register_microbatches; +use sqlbuild_python_planner::planner_strategy::main::strategy::register_strategy::register_strategy; use sqlbuild_python_project::planner_strategy::main::register_fixture_completion::register_fixture_completion; use sqlbuild_python_project::planner_strategy::main::register_fixture_context::register_fixture_context; use sqlbuild_python_project::planner_strategy::main::register_fixture_reads::register_fixture_reads; +use sqlbuild_python_planner::planner_strategy::main::audit_entries::plan_audit_of_function::plan_audit_of_function; +use sqlbuild_python_planner::planner_strategy::main::cursor_inputs::cursor_input_columns_of_function::cursor_input_columns_of_function; +use sqlbuild_python_planner::planner_strategy::main::cursor_inputs::cursor_input_relations_of_function::cursor_input_relations_of_function; +use sqlbuild_python_planner::planner_strategy::main::cursor_inputs::runtime_cursor_producer_names_function::runtime_cursor_producer_names_function; +use sqlbuild_python_planner::planner_strategy::main::microbatch_limits::apply_configured_microbatch_limits_function::apply_configured_microbatch_limits_function; +use sqlbuild_python_planner::planner_strategy::main::microbatch_limits::apply_microbatch_limit_function::apply_microbatch_limit_function; +use sqlbuild_python_planner::planner_strategy::main::model_entries::resolve_microbatch_lookback_function::resolve_microbatch_lookback_function; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_strategy::audit_entries::plan_audit_of, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_strategy::cursor_inputs::cursor_input_columns_of, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_strategy::cursor_inputs::cursor_input_relations_of, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_strategy::cursor_inputs::runtime_cursor_producer_names, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_strategy::model_entries::resolve_microbatch_lookback, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_microbatch_limit, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_configured_microbatch_limits, - module - )?)?; - audits::register(module)?; - cursor_types::register(module)?; + module.add_function(plan_audit_of_function(module)?)?; + module.add_function(cursor_input_columns_of_function(module)?)?; + module.add_function(cursor_input_relations_of_function(module)?)?; + module.add_function(runtime_cursor_producer_names_function(module)?)?; + module.add_function(resolve_microbatch_lookback_function(module)?)?; + module.add_function(apply_microbatch_limit_function(module)?)?; + module.add_function(apply_configured_microbatch_limits_function(module)?)?; + register_audits(module)?; + register_cursor_types(module)?; register_fixture_completion(module)?; register_fixture_context(module)?; register_fixture_reads(module)?; - microbatches::register(module)?; - strategy::register(module) + register_microbatches(module)?; + register_strategy(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs index 69d9ef9fe4..da41dfcf50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs @@ -1,14 +1,3 @@ //! Native warehouse inspection for planning over Python adapter callbacks. -pub(crate) mod constants; -pub(crate) mod cursors; -pub(crate) mod fingerprint_cache; -pub(crate) mod models; pub(crate) mod registration; -pub(crate) mod semantic_sources; -pub(crate) mod snapshot; -pub(crate) mod source_columns; -pub(crate) mod source_deferral; -pub(crate) mod source_freshness; -pub(crate) mod source_tables; -pub(crate) mod watermarks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/models.rs deleted file mode 100644 index 6c19c1fcea..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/models.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Warehouse planning state shared across planner runs. - -use std::collections::BTreeSet; - -use pyo3::pyclass; - -/// Migration fingerprints computed since the last reset, for work-count tests. -#[pyclass] -#[derive(Debug, Default)] -pub(crate) struct FingerprintWork { - pub(crate) computations: u64, - pub(crate) distinct: BTreeSet, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs index 5b7ca3db79..f2cf7b6049 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs @@ -1,44 +1,27 @@ //! Register native warehouse inspection for planning. use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::wrap_pyfunction; +use sqlbuild_python_planner::planner_warehouse::main::inspection::gather_warehouse_snapshot_function::gather_warehouse_snapshot_function; +use sqlbuild_python_planner::planner_warehouse::main::inspection::migration_fingerprint_work_info_function::migration_fingerprint_work_info_function; +use sqlbuild_python_planner::planner_warehouse::main::inspection::planner_metadata_name_filter_function::planner_metadata_name_filter_function; +use sqlbuild_python_planner::planner_warehouse::main::inspection::watermark_type_is_compatible_function::watermark_type_is_compatible_function; +use sqlbuild_python_planner::planner_warehouse::main::sources::build_planner_source_freshness_result_of_function::build_planner_source_freshness_result_of_function; +use sqlbuild_python_planner::planner_warehouse::main::sources::check_selected_source_tables_exist_binding_function::check_selected_source_tables_exist_binding_function; +use sqlbuild_python_planner::planner_warehouse::main::sources::gather_source_columns_function::gather_source_columns_function; +use sqlbuild_python_planner::planner_warehouse::main::sources::list_source_candidates_function::list_source_candidates_function; +use sqlbuild_python_planner::planner_warehouse::main::sources::semantic_source_columns_function::semantic_source_columns_function; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!( - super::fingerprint_cache::migration_fingerprint_work_info, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::snapshot::gather_warehouse_snapshot, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::watermarks::watermark_type_is_compatible, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::source_columns::gather_source_columns, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::snapshot::planner_metadata_name_filter, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::semantic_sources::list_source_candidates, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::semantic_sources::semantic_source_columns, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::source_tables::check_selected_source_tables_exist_binding, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::source_freshness::build_planner_source_freshness_result_of, - module - )?) + module + .add_class::( + )?; + module.add_function(migration_fingerprint_work_info_function(module)?)?; + module.add_function(gather_warehouse_snapshot_function(module)?)?; + module.add_function(watermark_type_is_compatible_function(module)?)?; + module.add_function(gather_source_columns_function(module)?)?; + module.add_function(planner_metadata_name_filter_function(module)?)?; + module.add_function(list_source_candidates_function(module)?)?; + module.add_function(semantic_source_columns_function(module)?)?; + module.add_function(check_selected_source_tables_exist_binding_function(module)?)?; + module.add_function(build_planner_source_freshness_result_of_function(module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index 42514a58ab..73a8bcc57d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -1,6 +1,5 @@ //! Native behaviour of the hosted shapes: computed attributes, methods, ordering and validation. -use crate::bindings::_helpers::freshness::observation::normalize_observation; use crate::bindings::_helpers::record_behaviours::auditing::{ finish_measurement_contract, finish_threshold_bound, finish_thresholds, }; @@ -71,13 +70,13 @@ use crate::bindings::_helpers::record_behaviours::values::{ }; use crate::bindings::_helpers::{ analysis, analysis_session, build_progress, compile_command, compute_logs, dbt_events, - dbt_interop, event_exporting, execution_history, planner_entries, planner_migrations, - planner_phases, project_assembly, result_output, rules, + dbt_interop, event_exporting, execution_history, project_assembly, result_output, rules, }; use sqlbuild_python_base::records::models::{GetterEntry, RecordBehaviours}; use sqlbuild_python_base::records::types::ClassMethod; use sqlbuild_python_base::records::types::ClassProperty; use sqlbuild_python_base::records::types::RecordFinish; +use sqlbuild_python_planner::freshness::main::normalize_observation::normalize_observation; use sqlbuild_python_project::lineage::main::facts_eq::facts_eq; use sqlbuild_python_project::lineage::main::facts_getitem::facts_getitem; use sqlbuild_python_project::lineage::main::facts_iter::facts_iter; @@ -372,9 +371,9 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ execution_history::getters::GETTERS, sqlbuild_python_project::graph::constants::GETTERS, sqlbuild_python_front::model_config::constants::GETTERS, - planner_entries::getters::GETTERS, - planner_migrations::getters::GETTERS, - planner_phases::getters::GETTERS, + sqlbuild_python_planner::planner_entries::constants::GETTERS, + sqlbuild_python_planner::planner_migrations::constants::GETTERS, + sqlbuild_python_planner::planner_phases::constants::GETTERS, sqlbuild_python_base::presentation_text::constants::GETTERS, project_assembly::getters::GETTERS, result_output::getters::GETTERS, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/freshness.rs index 14b1ae4809..13b1403a95 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/freshness.rs @@ -3,10 +3,10 @@ use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::types::PyDict; -use crate::bindings::_helpers::freshness::observation::require_aware_utc_datetime; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_planner::freshness::main::require_aware_utc_datetime::require_aware_utc_datetime; /// `SourceFreshnessRecord.observed_at` position. const OBSERVED_AT_FIELD: usize = 9; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs index f11dfcc336..f9b8acfd88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs @@ -8,12 +8,15 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::plan_output::safety::{future_cursor_safety, maximum_start_safety}; -use crate::bindings::_helpers::plan_output::values::{any, dict, items}; use crate::bindings::_helpers::result_output::projection::{Projection, rounded}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_planner::plan_output::main::any::any; +use sqlbuild_python_planner::plan_output::main::dict::dict; +use sqlbuild_python_planner::plan_output::main::future_cursor_safety::future_cursor_safety; +use sqlbuild_python_planner::plan_output::main::items::items; +use sqlbuild_python_planner::plan_output::main::maximum_start_safety::maximum_start_safety; /// A dictionary of the entries whose value is not `None`, in order. pub(crate) fn without_none<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs index 3b13e982ad..20dc6a7cc0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs @@ -9,11 +9,14 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::plan_output::values::{any, dict, items, text}; use crate::bindings::_helpers::result_output::assets::{or_none, without_none}; use crate::bindings::_helpers::result_output::projection::{Projection, ResultIdentity, rounded}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::scopes::main::reports::json_to_python::json_to_python; +use sqlbuild_python_planner::plan_output::main::any::any; +use sqlbuild_python_planner::plan_output::main::dict::dict; +use sqlbuild_python_planner::plan_output::main::items::items; +use sqlbuild_python_planner::plan_output::main::text::text; const DECIMAL_KIND: &str = "decimal"; const SCALAR_KINDS: [&str; 5] = ["string", "integer", "boolean", "float", "null"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs index ff60788c89..3ff857802d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs @@ -10,7 +10,6 @@ use sqlbuild_host::models::{ SqlTestExecutionResult, SqlTestOutcome, }; -use crate::bindings::_helpers::plan_output::values::{any, dict, items, text}; use crate::bindings::_helpers::result_output::aggregation::RetainedBuildResults; use crate::bindings::_helpers::result_output::assets::{ clone_asset, function_assets, load_assets, model_assets, python_node_assets, seed_assets, @@ -25,6 +24,10 @@ use sqlbuild_python_base::compile_outputs::main::indented_json::indented_json; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_planner::plan_output::main::any::any; +use sqlbuild_python_planner::plan_output::main::dict::dict; +use sqlbuild_python_planner::plan_output::main::items::items; +use sqlbuild_python_planner::plan_output::main::text::text; const JSON_VERSION: i64 = 1; const FAILED: &str = "failed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs index f820e6902f..8c098250dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs @@ -14,8 +14,6 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::plan_output::safety::{future_cursor_safety, maximum_start_safety}; -use crate::bindings::_helpers::plan_output::values::any; use crate::bindings::_helpers::result_output::assets::microbatch_summary; use crate::bindings::_helpers::result_output::checks::{ measured_value, measurement_thresholds, python_check_status, @@ -32,6 +30,9 @@ use sqlbuild_python_base::result_output::main::required_json::required_json; use sqlbuild_python_base::result_output::main::text::text; use sqlbuild_python_base::result_output::main::text_or_empty::text_or_empty; use sqlbuild_python_base::result_output::types::Checked; +use sqlbuild_python_planner::plan_output::main::any::any; +use sqlbuild_python_planner::plan_output::main::future_cursor_safety::future_cursor_safety; +use sqlbuild_python_planner::plan_output::main::maximum_start_safety::maximum_start_safety; const FAILED_EVENT: &str = "resource_attempt_failed"; const SKIPPED_EVENT: &str = "resource_attempt_skipped"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs index a35bae9289..d9228df633 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs @@ -11,11 +11,11 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::plan_output::values::text; use crate::bindings::_helpers::result_output::resource_ids::{audit_identity, sql_test_identity}; use crate::bindings::_helpers::result_output::terminal_index::{TerminalEventIndex, current_index}; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_planner::plan_output::main::text::text; /// The executor result types that result output distinguishes. #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 95b3ab4c7e..28c9cb55df 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -23,8 +23,6 @@ use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pi use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::planner_entries::command_plans::build_seed_command_plan; -use crate::bindings::_helpers::planner_entries::command_plans::read_selected_seed_fingerprints; use crate::bindings::_helpers::result_output::documents::format_seed_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; @@ -38,6 +36,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_planner::planner_entries::main::plans::build_seed_command_plan::build_seed_command_plan; +use sqlbuild_python_planner::planner_entries::main::plans::read_selected_seed_fingerprints::read_selected_seed_fingerprints; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; use sqlbuild_python_project::project_snapshot::types::VerifiedRun; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 7b6a2236b4..1b5dbd9490 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -41,7 +41,6 @@ use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pi use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runtime_target; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::planner_entries::command_plans::build_test_command_plan; use crate::bindings::_helpers::result_output::documents::test_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; @@ -69,6 +68,7 @@ use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_planner::planner_entries::main::plans::build_test_command_plan::build_test_command_plan; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; use sqlbuild_python_project::project_snapshot::types::VerifiedRun; diff --git a/fensu.toml b/fensu.toml index a7413cae2b..f0d95e486e 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1672,12 +1672,12 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 147491 } +thresholds = { max_crate_code_lines = 114941 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/src/bindings/_helpers"] -thresholds = { max_helpers_container_buckets = 98 } +thresholds = { max_helpers_container_buckets = 96 } reason = "The bindings helpers container is above the bucket limit; this caps it at its current bucket count until cohesive groups move into their own modules or crates. Lower the cap as buckets move out." [targets.rust.rule_options.XSB076] @@ -1704,7 +1704,7 @@ allowed_paths = [ "crates/sqlbuild-python-front/src/native_host/main/append_observations.rs=Writes observations after a build.", "crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs=Registers the observation state bindings.", "crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs=Registers the observation state bindings.", - "crates/sqlbuild-python/src/bindings/_helpers/observations=Writes observations after a build.", + "crates/sqlbuild-python-planner/src/observations=Writes observations after a build.", "crates/sqlbuild-python-front/src/native_host/main/mod.rs=Declares the observation write and registration entries.", ] @@ -1712,7 +1712,7 @@ allowed_paths = [ allowed_paths = [ "crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs=A location's requested plan names the synced copy for its sqb plan child.", "crates/sqlbuild-core/src/plan_display/constants.rs=Names the synced-copy environment variable.", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs=Preview branch: a refused freshness observation leaves the source unknown.", + "crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs=Preview branch: a refused freshness observation leaves the source unknown.", "crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs=Registers the synced copy's refusal error with Python.", "crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs=Defines the synced-copy adapter.", "crates/sqlbuild-python-base/src/warehouse_access/main/register_synced_copy.rs=Registers the synced copy's refusal error with Python.", @@ -1720,12 +1720,12 @@ allowed_paths = [ "crates/sqlbuild-python-base/src/warehouse_access/main/is_warehouse_access_refused.rs=Tells a plan preview's refused warehouse access apart.", "crates/sqlbuild-python-base/src/warehouse_access/models.rs=Defines the synced-copy adapter the plan preview reads through.", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=Only sqb plan activates the synced copy and reports what it could not know.", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=Preview branch: target MAX from the copy, watermark inputs read when the run starts.", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=Preview branch: table types and retention stay unknown.", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs=Preview branch: cursor intrinsics bind to bounds the run resolves.", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs=Preview branch: expression columns from the copy instead of DESCRIBE.", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs=Preview branch: the copy has no source freshness state to read.", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs=Preview branch: sources outside the copy are not probed.", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs=Preview branch: target MAX from the copy, watermark inputs read when the run starts.", + "crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs=Preview branch: table types and retention stay unknown.", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs=Preview branch: cursor intrinsics bind to bounds the run resolves.", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs=Preview branch: expression columns from the copy instead of DESCRIBE.", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs=Preview branch: the copy has no source freshness state to read.", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs=Preview branch: sources outside the copy are not probed.", "crates/sqlbuild-python-front/src/native_host/models.rs=Preview branch: --changed reads consumption history from the copy instead of the warehouse table.", ] @@ -2024,6 +2024,7 @@ crate_names = [ "sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", + "sqlbuild-python-planner", "sqlbuild-python", "sqlbuild-recall", "sqlbuild-refactor", @@ -2049,7 +2050,7 @@ paths = ["src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py"] reason = "SQL Server overrides each state table's create SQL to add that table's own key index; the keyed DDL lives in _keyed_state_table_sql." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs", "crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs"] +paths = ["crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs", "crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs"] reason = "Each turns its own record type into its own Python objects; only the loop that collects them is alike." [[dupes.allowlist]] @@ -2272,6 +2273,102 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs=1", "crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs=1", "crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs=1", + "crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs=6", + "crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs=3", + "crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs=1", + "crates/sqlbuild-python-planner/src/freshness/_helpers/publication.rs=9", + "crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs=3", + "crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs=2", + "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs=4", + "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/payloads.rs=4", + "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs=4", + "crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs=1", + "crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs=1", + "crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs=5", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs=5", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs=10", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs=1", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs=7", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs=5", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/python_identity.rs=2", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs=1", + "crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs=5", + "crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs=2", + "crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs=4", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs=2", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs=1", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/values.rs=1", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs=5", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs=4", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs=1", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs=2", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs=2", + "crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs=2", + "crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs=6", + "crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs=3", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs=3", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs=8", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs=6", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_models.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs=9", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs=5", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs=4", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs=13", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs=4", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs=7", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs=7", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs=6", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/projections.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/models.rs=1", + "crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs=2", + "crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs=1", + "crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs=3", + "crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs=2", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_bounds.rs=2", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs=3", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs=2", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_policies.rs=2", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs=5", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_safety.rs=2", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_windows.rs=1", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs=1", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs=4", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs=1", + "crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs=3", + "crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs=1", + "crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs=1", + "crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs=1", + "crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs=1", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs=6", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs=8", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs=2", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs=1", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs=3", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs=1", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs=14", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs=6", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs=3", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs=2", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs=3", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs=1", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs=1", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs=6", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs=5", + "crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs=1", + "crates/sqlbuild-python-planner/src/planner_warehouse/models.rs=8", "crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs=0", "crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs=1", "crates/sqlbuild-python-project/src/analysis/models.rs=1", @@ -2500,12 +2597,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs=17", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs=3", @@ -2540,9 +2631,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs=12", @@ -2556,101 +2644,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=31", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs=3", @@ -2811,6 +2810,30 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/scopes/models.rs=11", "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs=1", "crates/sqlbuild-python-front/src/state_rows/models.rs=3", + "crates/sqlbuild-python-planner/src/freshness/models.rs=8", + "crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs=1", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs=2", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs=2", + "crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs=1", + "crates/sqlbuild-python-planner/src/planner_changes/models.rs=3", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs=6", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs=9", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs=2", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs=2", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs=4", + "crates/sqlbuild-python-planner/src/planner_entries/models.rs=56", + "crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs=4", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/known_input_columns.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/models.rs=77", + "crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs=4", + "crates/sqlbuild-python-planner/src/planner_phases/models.rs=38", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs=3", + "crates/sqlbuild-python-planner/src/planner_resolution/models.rs=20", + "crates/sqlbuild-python-planner/src/planner_scenario/models.rs=9", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs=1", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs=19", + "crates/sqlbuild-python-planner/src/planner_warehouse/models.rs=11", "crates/sqlbuild-python-project/src/analysis/models.rs=9", "crates/sqlbuild-python-project/src/analysis_session/models.rs=1", "crates/sqlbuild-python-project/src/compiled_project/models.rs=36", @@ -2902,9 +2925,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs=22", @@ -2931,35 +2951,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=56", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs=66", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=38", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=19", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=3", @@ -3060,14 +3055,14 @@ module_layers = [ "init_command=commands", "input_loops=compile", "janitor=commands", - "lifecycle_events=plan", + "lifecycle_events=registry", "lineage=compile", "lineage/lineage_command=commands", "load=commands", "location_command=commands", "macro_bridge=registry", "manifest=compile", - "microbatches=plan", + "microbatches=registry", "model_config=registry", "native_host=front", "native_host/orchestration_sensors=commands", @@ -3076,20 +3071,19 @@ module_layers = [ "observations=plan", "output_capture=output", "plan_command=commands", - "plan_output=plan", - "planner_changes=plan", - "planner_entries=plan", - "planner_identity=plan", - "planner_migrations=plan", - "planner_phases=plan", - "planner_phases/models=project", + "plan_output=registry", + "planner_changes=registry", + "planner_entries=registry", + "planner_identity=registry", + "planner_migrations=registry", + "planner_phases=registry", "planner_registration=registry", - "planner_resolution=plan", - "planner_scenario=plan", + "planner_resolution=registry", + "planner_scenario=registry", "planner_scope=registry", "planner_selection=registry", - "planner_strategy=plan", - "planner_warehouse=plan", + "planner_strategy=registry", + "planner_warehouse=registry", "presentation_text=registry", "project_assembly=compile", "query_command=commands", @@ -3159,7 +3153,7 @@ paths = ["crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs", "c reason = "Field lookups on unrelated value types in different crates: project configuration takes the first matching field of a built object, while rule headers take the last entry of an authored map so a repeated key overrides earlier ones." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs", "crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs"] +paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs", "crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs"] reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." [[dupes.allowlist]] @@ -3171,7 +3165,7 @@ paths = ["crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registra reason = "Each bindings module registers its own functions and classes with the native module." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/diagnostics/_helpers/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/getters.rs"] +paths = ["crates/sqlbuild-python-base/src/diagnostics/_helpers/getters.rs", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs"] reason = "Direct `#[getter]` readers of each module's own pyclass; each matches its own class and field names." [[dupes.allowlist]] @@ -3195,11 +3189,11 @@ paths = ["crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs", "crates reason = "Each maps a Python `str` into its own JSON value type; the UTF-16 conversion itself is shared through `utf16_units`." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/warehouse_access/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs"] +paths = ["crates/sqlbuild-python-base/src/warehouse_access/models.rs", "crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs"] reason = "Each turns its own record type into its own Python objects; only the loop that collects them is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs"] +paths = ["crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs", "crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs"] reason = "The capture encoder sorts by canonical capture key and freshness planning by resource identity; only the decorate-sort-undecorate shape is alike." [[dupes.allowlist]] @@ -3231,7 +3225,7 @@ paths = ["crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs", reason = "Each reads its own Python record's fields into its own native record." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs", "crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs"] +paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs", "crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs"] reason = "Each bindings module registers its own functions and classes with the native module." [[dupes.allowlist]] @@ -3263,7 +3257,7 @@ paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/model_input reason = "Each borrows its own record from the compiled project owner by index; only the borrow-and-read shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs"] +paths = ["crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/python_identity.rs"] reason = "Each reads an optional text value from its own Python mapping; consolidating these readers across the binding crates is a separate change." [[dupes.allowlist]] @@ -3299,7 +3293,7 @@ paths = ["crates/sqlbuild-python-project/src/project_snapshot/models.rs"] reason = "Each worker report forwards one distinct check event with its own fields." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs"] +paths = ["crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs"] reason = "One returns the first model destination's database, the other collects every model destination's schema; only the loop over compiled models is alike." [[dupes.allowlist]] @@ -3335,19 +3329,19 @@ paths = ["crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs" reason = "Each public entry forwards to its own helper with the same arguments." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs", "crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs"] reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs"] +paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs"] reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs", "crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs"] +paths = ["crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs", "crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs"] reason = "Each bindings module registers its own functions and classes with the native module." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs"] +paths = ["crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/python_identity.rs"] reason = "Each reads an optional text value from its own Python mapping; consolidating these readers across the binding crates is a separate change." [[dupes.allowlist]] @@ -3391,7 +3385,7 @@ paths = ["crates/sqlbuild-python-project/src/resource_views/main/audit_reads/alw reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/oracles.rs", "crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs", "crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs", "crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs", "crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/oracles.rs", "crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs", "crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs", "crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs", "crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs", "crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3399,15 +3393,15 @@ paths = ["crates/sqlbuild-python-front/src/attachments/main/register_cursor_intr reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs"] +paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs"] reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs"] +paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs"] +paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3415,11 +3409,11 @@ paths = ["crates/sqlbuild-python-project/src/resource_views/main/resource_reads/ reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs", "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs"] +paths = ["crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs", "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs"] reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs"] +paths = ["crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs"] reason = "Each reads the installed package version for its own output with its own fallback; only the metadata lookup is alike." [[dupes.allowlist]] @@ -3439,7 +3433,7 @@ paths = ["crates/sqlbuild-python-project/src/compiled_views/main/project/audits. reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs"] +paths = ["crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs"] reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." [[dupes.allowlist]] @@ -3475,7 +3469,7 @@ paths = ["crates/sqlbuild-python-front/src/model_config/_helpers/config/config_v reason = "Each checks one config key as its own value type; only the node lookup and error mapping are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/python_calls.rs", "crates/sqlbuild-python-project/src/planner_phases/main/replace.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/python_calls.rs", "crates/sqlbuild-python-project/src/planner_phases/main/replace.rs", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/values.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_safety.rs"] reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." [[dupes.allowlist]] @@ -3563,7 +3557,7 @@ paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migrat reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/planner_scope/main/edges.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs"] +paths = ["crates/sqlbuild-python-project/src/planner_scope/main/edges.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/payloads.rs"] reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] @@ -3583,7 +3577,7 @@ paths = ["crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.r reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs"] +paths = ["crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs", "crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs"] reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] @@ -3611,7 +3605,7 @@ paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshn reason = "Each builds its own error class's exception from a message; sharing one exception constructor across the binding crates is a separate change." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs"] +paths = ["crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs"] reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 3c7b02624c..695d7a15a3 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -288,6 +288,7 @@ "crates/sqlbuild-python-base/src/", "crates/sqlbuild-python-front/src/", "crates/sqlbuild-python-project/src/", + "crates/sqlbuild-python-planner/src/", ) NATIVE_CROSSING_PATTERNS: tuple[str, ...] = ( r"\.getattr\(", diff --git a/scripts/native_module_paths/constants.py b/scripts/native_module_paths/constants.py index 904f3f4ce6..e426c4f2b7 100644 --- a/scripts/native_module_paths/constants.py +++ b/scripts/native_module_paths/constants.py @@ -6,6 +6,7 @@ "crates/sqlbuild-python-base/src", "crates/sqlbuild-python-front/src", "crates/sqlbuild-python-project/src", + "crates/sqlbuild-python-planner/src", "crates/sqlbuild-python/src", "crates/sqlbuild-cli/src", "crates/sqlbuild-host/src", From 99bf0842595d05a352b4b93c130ffbc28a26b76f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 18:25:44 +0000 Subject: [PATCH 3/5] refactor: fold planner forwarders and regroup imports --- .../src/consumption/_helpers/facts.rs | 5 +- .../src/consumption/_helpers/policy.rs | 104 +------- .../src/consumption/_helpers/recorder.rs | 63 +---- .../consumption/_helpers/relation_reads.rs | 9 - .../src/consumption/main/changed_policy.rs | 58 ++++- .../src/consumption/main/held_back_text.rs | 10 +- .../src/consumption/main/kind_value.rs | 11 +- .../src/consumption/main/policy_line.rs | 22 +- .../src/consumption/main/prepare.rs | 4 +- .../src/consumption/main/preview_changed.rs | 24 +- .../src/consumption/main/reach_value.rs | 11 +- .../src/consumption/main/relation.rs | 14 +- .../src/consumption/main/run_reason_text.rs | 25 +- .../src/freshness/_helpers/exception_types.rs | 13 - .../freshness/_helpers/freshness_observe.rs | 21 +- .../src/freshness/_helpers/observation.rs | 66 +---- .../src/freshness/_helpers/planning.rs | 28 +-- .../src/freshness/_helpers/publication.rs | 69 +----- .../src/freshness/_helpers/state_io.rs | 68 +----- .../src/freshness/_helpers/support.rs | 22 +- .../main/append_freshness_records.rs | 59 ++++- .../freshness/main/normalize_observation.rs | 27 ++- .../src/freshness/main/observe_freshness.rs | 4 +- .../freshness/main/presumed_utc_datetime.rs | 18 +- .../src/freshness/main/read_latest.rs | 4 +- .../main/register_exception_types.rs | 18 +- .../main/register_freshness_observe.rs | 11 +- .../freshness/main/register_observation.rs | 16 +- .../src/freshness/main/register_planning.rs | 19 +- .../freshness/main/register_publication.rs | 13 +- .../src/freshness/main/register_state_io.rs | 23 +- .../main/require_aware_utc_datetime.rs | 17 +- .../src/freshness/main/row_record.rs | 44 +++- .../src/lifecycle_events/_helpers/catalog.rs | 51 +--- .../lifecycle_events/_helpers/envelopes.rs | 38 +-- .../lifecycle_events/_helpers/validation.rs | 19 +- .../lifecycle_events/main/register_catalog.rs | 49 +++- .../main/register_envelopes.rs | 22 +- .../main/register_validation.rs | 17 +- .../src/microbatches/_helpers/conversion.rs | 5 - .../src/microbatches/_helpers/projection.rs | 22 +- .../src/microbatches/_helpers/statements.rs | 30 +-- .../microbatches/main/register_projection.rs | 14 +- .../microbatches/main/register_statements.rs | 20 +- .../src/microbatches/main/timestamp.rs | 8 +- .../src/observations/_helpers/publication.rs | 42 +--- .../observations/_helpers/source_capture.rs | 38 +-- .../main/append_build_observations.rs | 41 +++- .../observations/main/arm_source_capture.rs | 5 +- .../main/capture_source_columns.rs | 23 +- .../observations/main/take_source_capture.rs | 7 +- .../plan_output/_helpers/cursor_details.rs | 6 +- .../src/plan_output/_helpers/display.rs | 145 +---------- .../src/plan_output/_helpers/migrations.rs | 25 +- .../src/plan_output/_helpers/plan_json.rs | 164 +++---------- .../src/plan_output/_helpers/plan_text.rs | 61 ++--- .../src/plan_output/_helpers/safety.rs | 146 +---------- .../src/plan_output/_helpers/values.rs | 37 +-- .../main/aligned_timestamp_bounds.rs | 21 +- .../src/plan_output/main/any.rs | 9 +- .../plan_output/main/cursor_bound_display.rs | 30 ++- .../src/plan_output/main/dict.rs | 15 +- .../src/plan_output/main/format_plan.rs | 28 ++- .../src/plan_output/main/format_plan_json.rs | 120 ++++++++- .../plan_output/main/future_cursor_safety.rs | 81 ++++++- .../plan_output/main/inclusive_cursor_end.rs | 19 +- .../src/plan_output/main/items.rs | 8 +- .../plan_output/main/maximum_start_safety.rs | 55 ++++- .../src/plan_output/main/optional_facts.rs | 12 +- .../src/plan_output/main/register_display.rs | 22 +- .../plan_output/main/register_migrations.rs | 12 +- .../plan_output/main/register_plan_json.rs | 10 +- .../plan_output/main/register_plan_text.rs | 10 +- .../src/plan_output/main/register_safety.rs | 10 +- .../main/resolve_bounded_cursor_override.rs | 32 ++- ...resolve_effective_microbatch_batch_size.rs | 19 +- .../src/plan_output/main/text.rs | 8 +- .../src/planner_changes/_helpers/actions.rs | 61 +---- .../src/planner_changes/_helpers/detect.rs | 69 +----- .../src/planner_changes/_helpers/metadata.rs | 24 +- .../src/planner_changes/_helpers/renames.rs | 10 +- .../src/planner_changes/_helpers/results.rs | 21 +- .../planner_changes/main/detect_changes.rs | 4 +- .../main/detect_model_changes.rs | 49 +++- .../src/planner_changes/main/model_change.rs | 4 +- .../src/planner_changes/main/native_enum.rs | 10 +- .../main/plan_reference_renames.rs | 7 +- .../src/planner_changes/main/planner_enum.rs | 10 +- .../planner_changes/main/register_actions.rs | 10 +- .../planner_changes/main/register_detect.rs | 14 +- .../planner_changes/main/register_metadata.rs | 14 +- .../planner_changes/main/register_renames.rs | 10 +- .../main/replaces_incremental_relation.rs | 9 +- .../main/resolve_model_actions.rs | 42 +++- .../main/scope_model_changes.rs | 4 +- .../main/without_declared_columns_hash.rs | 7 +- .../_helpers/entries/contexts.rs | 14 +- .../_helpers/entries/getters.rs | 6 +- .../_helpers/entries/model_entries.rs | 115 ++------- .../_helpers/entries/relations.rs | 13 +- .../_helpers/plans/clone_plans.rs | 16 +- .../_helpers/plans/command_plans.rs | 150 +----------- .../_helpers/plans/display_plan.rs | 67 +---- .../_helpers/plans/execution_plan.rs | 98 +------- .../_helpers/plans/plan_output.rs | 21 +- .../_helpers/plans/test_planning.rs | 21 +- .../default_microbatch_limit_action.rs | 11 +- .../main/entries/plan_model.rs | 72 +++++- .../main/entries/planner_relations_context.rs | 4 +- .../main/entries/register_model_entries.rs | 12 +- .../main/entries/register_relations.rs | 10 +- .../main/plans/build_audit_command_plan.rs | 54 ++++- .../plans/build_display_only_sqlbuild_plan.rs | 54 ++++- .../main/plans/build_execution_plan.rs | 82 ++++++- .../main/plans/build_relation_command_plan.rs | 10 +- .../main/plans/build_seed_command_plan.rs | 4 +- .../main/plans/build_test_command_plan.rs | 5 +- .../main/plans/plan_output_fields.rs | 4 +- .../plans/read_selected_seed_fingerprints.rs | 4 +- .../main/plans/register_clone_plans.rs | 12 +- .../main/plans/register_command_plans.rs | 20 +- .../main/plans/register_display_plan.rs | 10 +- .../main/plans/register_execution_plan.rs | 10 +- .../main/plans/register_plan_output.rs | 12 +- .../main/plans/register_test_planning.rs | 18 +- .../plans/resolve_static_command_scope.rs | 24 +- .../plans/resolve_static_relation_context.rs | 40 ++- .../main/plans/timed_clone_planning.rs | 7 +- .../src/planner_entries/models.rs | 4 +- .../planner_identity/_helpers/identities.rs | 61 +---- .../src/planner_identity/_helpers/metadata.rs | 194 +-------------- .../planner_identity/_helpers/staleness.rs | 120 +-------- ...build_compiled_function_fingerprint_sql.rs | 11 +- .../main/build_function_local_hashes.rs | 16 +- ...ld_model_version_identity_metadata_json.rs | 16 +- .../main/build_seed_identity.rs | 11 +- .../build_stale_out_of_selection_warnings.rs | 96 +++++++- .../main/compiled_function_fingerprint_sql.rs | 36 ++- .../main/direct_model_version_identities.rs | 4 +- .../main/function_local_hashes.rs | 4 +- .../main/hook_version_hashes.rs | 22 +- .../main/merge_recomputed_model_changes.rs | 29 ++- .../main/model_declared_columns_hash.rs | 45 +++- .../model_version_identity_metadata_json.rs | 47 +++- .../src/planner_identity/main/or_empty.rs | 13 +- .../main/register_identities.rs | 15 +- .../main/register_metadata.rs | 29 ++- .../main/register_staleness.rs | 18 +- .../planner_identity/main/seed_identity.rs | 4 +- .../main/with_honest_model_write_hashes.rs | 5 +- .../_helpers/column/column_candidates.rs | 8 +- .../_helpers/column/column_entries.rs | 10 +- .../_helpers/column/column_overlay.rs | 11 +- .../_helpers/column/column_planning.rs | 16 +- .../_helpers/column/column_warnings.rs | 8 +- .../_helpers/column/columns.rs | 9 +- .../_helpers/discovery/discovery.rs | 11 +- .../_helpers/discovery/discovery_requests.rs | 10 +- .../_helpers/model/model_entries.rs | 14 +- .../_helpers/model/model_handover.rs | 3 +- .../_helpers/model/model_overlay.rs | 3 +- .../_helpers/model/model_planning.rs | 40 +-- .../_helpers/model/model_warnings.rs | 5 +- .../_helpers/old_name/old_name_entries.rs | 14 +- .../_helpers/old_name/old_name_objects.rs | 3 +- .../_helpers/old_name/old_name_planning.rs | 3 +- .../_helpers/old_name/old_name_references.rs | 3 +- .../_helpers/planning/compatibility.rs | 3 +- .../_helpers/planning/matching.rs | 3 +- .../_helpers/projections/fingerprint.rs | 84 +------ .../_helpers/projections/projections.rs | 9 +- .../planner_migrations/main/fingerprint.rs | 14 +- .../main/migration_fingerprint_ref_names.rs | 14 +- .../main/plan_column_migrations.rs | 6 +- .../main/plan_model_migrations.rs | 23 +- .../main/plan_old_name_views.rs | 7 +- .../main/register_column_planning.rs | 10 +- .../main/register_columns.rs | 12 +- .../main/register_fingerprint.rs | 11 +- .../main/register_projections.rs | 12 +- .../main/with_migration_fingerprints.rs | 38 ++- .../src/planner_migrations/models.rs | 3 +- .../planner_phases/_helpers/inputs/getters.rs | 6 +- .../planner_phases/_helpers/inputs/runtime.rs | 3 +- .../_helpers/inputs/warehouse_state.rs | 5 +- .../_helpers/outputs/identities.rs | 46 +--- .../_helpers/outputs/output_assembly.rs | 131 ++-------- .../_helpers/rebuilds/execution_changes.rs | 44 +--- .../_helpers/rebuilds/full_refresh.rs | 107 +------- .../_helpers/rebuilds/retention.rs | 52 +--- .../planner_phases/main/inputs/class_field.rs | 5 +- .../inputs/gather_planner_warehouse_state.rs | 4 +- .../main/inputs/runtime_field.rs | 5 +- .../main/inputs/warehouse_field.rs | 5 +- .../main/outputs/add_deferred_origin_notes.rs | 69 +++++- .../main/outputs/add_plan_metadata.rs | 30 ++- .../main/outputs/add_plan_warnings.rs | 7 +- .../main/outputs/add_storage_policies.rs | 3 +- .../main/outputs/base_plan_output.rs | 8 +- .../outputs/build_planner_identity_context.rs | 37 ++- .../check_selected_scope_buildability.rs | 4 +- .../outputs/detect_stale_warning_changes.rs | 4 +- ...etect_stale_warning_changes_of_function.rs | 5 +- .../planner_identity_work_info_function.rs | 5 +- .../outputs/unselected_upstream_models.rs | 26 +- .../rebuilds/build_planner_entry_results.rs | 7 +- .../rebuilds/check_full_rebuild_allowed.rs | 39 ++- .../check_full_rebuild_allowed_function.rs | 5 +- .../rebuilds/defer_to_migration_origins.rs | 4 +- .../rebuilds/describe_full_rebuild_cause.rs | 24 +- .../describe_full_rebuild_cause_function.rs | 5 +- .../main/rebuilds/effective_target_config.rs | 20 +- .../effectively_full_refreshed_model_names.rs | 19 +- .../main/rebuilds/plan_retention_function.rs | 5 +- .../rebuilds/plan_table_types_function.rs | 5 +- .../rebuilds/reconcile_execution_changes.rs | 40 ++- .../rebuilds/resolve_model_full_refresh.rs | 14 +- .../rebuilds/table_type_copy_name_function.rs | 5 +- .../src/planner_phases/models.rs | 4 +- .../_helpers/cursor/cursor_bounds.rs | 9 +- .../_helpers/cursor/cursor_model.rs | 5 +- .../_helpers/cursor/cursor_overrides.rs | 15 +- .../_helpers/cursor/cursor_policies.rs | 8 - .../_helpers/cursor/cursor_ranges.rs | 229 +++--------------- .../_helpers/cursor/cursor_warnings.rs | 55 +---- .../_helpers/cursor/cursor_windows.rs | 52 +--- .../_helpers/references/locations.rs | 30 +-- .../_helpers/references/orchestration.rs | 84 +------ .../_helpers/references/refs.rs | 55 +---- .../_helpers/references/sources.rs | 21 +- .../_helpers/safety/future_safety.rs | 17 +- .../_helpers/safety/maximum_safety.rs | 68 +----- .../main/cursor/config_str.rs | 12 +- .../main/cursor/cursor_safety_configs.rs | 22 +- .../main/cursor/empty_cursor_inputs.rs | 35 ++- .../main/cursor/microbatch_range.rs | 4 +- .../main/cursor/model_cursor_overrides.rs | 74 +++++- .../cursor/normalize_cursor_snapshot_grain.rs | 4 +- .../main/cursor/plan_cursor_bounds.rs | 51 +++- .../main/cursor/register_cursor_bounds.rs | 12 +- .../main/cursor/register_cursor_overrides.rs | 10 +- .../main/cursor/register_cursor_ranges.rs | 14 +- .../main/cursor/register_cursor_warnings.rs | 12 +- .../main/cursor/register_cursor_windows.rs | 20 +- .../resolve_effective_timestamp_grain.rs | 43 +++- .../cursor/resolve_start_cursor_config.rs | 9 +- .../main/cursor/type_cursor_overrides.rs | 4 +- .../cursor/with_cursor_safety_warnings.rs | 40 ++- .../references/apply_deferred_locations.rs | 37 ++- .../references/build_function_locations.rs | 12 +- .../main/references/build_model_locations.rs | 12 +- .../main/references/build_seed_locations.rs | 12 +- .../main/references/model_hook_entries.rs | 21 +- .../main/references/register_locations.rs | 14 +- .../main/references/register_orchestration.rs | 14 +- .../main/references/register_refs.rs | 19 +- .../main/references/register_sources.rs | 13 +- .../main/references/resolve_function_sql.rs | 50 +++- .../main/references/resolve_model_sql.rs | 4 +- .../main/safety/maximum_allowed_start.rs | 50 +++- .../main/safety/register_future_safety.rs | 10 +- .../main/safety/register_maximum_safety.rs | 12 +- .../src/planner_resolution/models.rs | 3 +- .../planner_scenario/_helpers/artifacts.rs | 57 +---- .../src/planner_scenario/_helpers/cli.rs | 7 - .../planner_scenario/_helpers/execution.rs | 20 +- .../src/planner_scenario/_helpers/fixtures.rs | 7 - .../src/planner_scenario/_helpers/graph.rs | 7 - .../planner_scenario/_helpers/relations.rs | 15 +- .../src/planner_scenario/_helpers/sql.rs | 12 +- .../main/build_cli_scenario_plan_function.rs | 5 +- .../build_scenario_artifact_name_function.rs | 5 +- .../build_scenario_execution_plan_function.rs | 5 +- .../build_scenario_fixture_plans_function.rs | 5 +- .../build_scenario_hash_index_function.rs | 5 +- .../build_scenario_relation_map_function.rs | 5 +- ...uild_scenario_relation_plan_of_function.rs | 5 +- .../compute_scenario_hash_prefix_function.rs | 5 +- .../is_scenario_artifact_physical_name.rs | 8 +- ...cenario_artifact_physical_name_function.rs | 5 +- .../main/plan_scenario_graph.rs | 4 +- .../main/plan_scenario_graph_function.rs | 5 +- .../resolve_scenario_check_sql_function.rs | 5 +- ...o_artifact_physical_name_check_function.rs | 5 +- .../src/planner_scenario/models.rs | 3 +- .../_helpers/audit_entries.rs | 81 +------ .../src/planner_strategy/_helpers/audits.rs | 14 +- .../_helpers/cursor_inputs.rs | 55 +---- .../planner_strategy/_helpers/cursor_types.rs | 45 +--- .../src/planner_strategy/_helpers/lineage.rs | 74 +----- .../_helpers/microbatch_limits.rs | 67 +---- .../planner_strategy/_helpers/microbatches.rs | 15 +- .../_helpers/model_entries.rs | 49 +--- .../src/planner_strategy/_helpers/strategy.rs | 85 +------ .../src/planner_strategy/_helpers/values.rs | 32 +-- .../main/audit_entries/audit_is_selected.rs | 31 ++- .../build_selected_audit_entries.rs | 44 +++- .../audit_entries/plan_audit_of_function.rs | 5 +- .../main/audits/register_audits.rs | 14 +- .../build_cursor_input_relations.rs | 4 +- .../cursor_input_columns_of_function.rs | 5 +- .../cursor_input_relations_of_function.rs | 5 +- .../reject_unexposed_cursor_input_columns.rs | 4 +- .../runtime_cursor_producer_names.rs | 29 ++- .../runtime_cursor_producer_names_function.rs | 5 +- .../cursor_types/classify_cursor_sql_type.rs | 29 ++- .../cursor_types/register_cursor_types.rs | 10 +- .../lineage/reject_capped_watermark_inputs.rs | 41 +++- .../main/lineage/resolve_lineage_reference.rs | 20 +- .../apply_configured_microbatch_limits.rs | 44 +++- ...y_configured_microbatch_limits_function.rs | 5 +- .../apply_microbatch_limit_function.rs | 5 +- .../microbatches/register_microbatches.rs | 11 +- .../assemble_model_plan_entry.rs | 4 +- .../model_entries/extract_seed_columns.rs | 9 +- .../resolve_microbatch_lookback.rs | 12 +- .../resolve_microbatch_lookback_function.rs | 5 +- .../main/strategy/materialization_of.rs | 18 +- .../main/strategy/register_strategy.rs | 16 +- .../strategy/resolve_model_plan_action.rs | 52 +++- .../main/values/input_error.rs | 19 +- .../src/planner_strategy/main/values/text.rs | 8 +- .../_helpers/inspection/cursors.rs | 73 +----- .../_helpers/inspection/fingerprint_cache.rs | 7 - .../_helpers/inspection/snapshot.rs | 27 +-- .../_helpers/inspection/watermarks.rs | 13 +- .../_helpers/sources/semantic_sources.rs | 17 +- .../_helpers/sources/source_columns.rs | 40 +-- .../_helpers/sources/source_deferral.rs | 30 +-- .../_helpers/sources/source_freshness.rs | 58 +---- .../_helpers/sources/source_tables.rs | 13 +- .../src/planner_warehouse/constants.rs | 3 +- .../main/inspection/adapter_cursor_sql.rs | 34 ++- .../main/inspection/bound_task.rs | 24 +- .../gather_redirected_cursor_snapshots.rs | 4 +- .../gather_warehouse_snapshot_function.rs | 5 +- ...igration_fingerprint_work_info_function.rs | 5 +- .../planner_metadata_name_filter_function.rs | 5 +- .../main/inspection/warehouse_snapshot.rs | 4 +- .../watermark_type_is_compatible_function.rs | 5 +- .../build_planner_source_freshness_result.rs | 4 +- ...ner_source_freshness_result_of_function.rs | 5 +- .../main/sources/build_source_read_map.rs | 6 +- .../check_selected_source_tables_exist.rs | 4 +- ...ed_source_tables_exist_binding_function.rs | 5 +- .../main/sources/gather_source_columns.rs | 13 +- .../sources/gather_source_columns_function.rs | 5 +- .../main/sources/gather_source_inspection.rs | 11 +- .../list_source_candidates_function.rs | 5 +- .../main/sources/semantic_source_columns.rs | 6 +- .../semantic_source_columns_function.rs | 5 +- .../source_freshness_unknown_warnings.rs | 52 +++- .../sources/with_declared_source_reads.rs | 32 ++- .../_helpers/build_command/execution.rs | 3 +- fensu.toml | 37 ++- 355 files changed, 3649 insertions(+), 5261 deletions(-) diff --git a/crates/sqlbuild-python-planner/src/consumption/_helpers/facts.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/facts.rs index c04e35d4a0..ac4fe2ff2a 100644 --- a/crates/sqlbuild-python-planner/src/consumption/_helpers/facts.rs +++ b/crates/sqlbuild-python-planner/src/consumption/_helpers/facts.rs @@ -1,5 +1,6 @@ //! The facts `--changed` and the consumption recorder read from the plan and compiled project. +use crate::consumption::main::relation::relation; use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -7,9 +8,7 @@ use pyo3::types::PyDict; use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; use sqlbuild_state::consumption::models::{ForcedRun, InputFact, NodeFacts, NodeKey, RelationKey}; -use crate::consumption::_helpers::relation_reads::Locations; -use crate::consumption::_helpers::relation_reads::relation; -use crate::consumption::_helpers::relation_reads::sql_reads_undeclared; +use crate::consumption::_helpers::relation_reads::{Locations, sql_reads_undeclared}; use crate::planner_identity::main::function_local_hashes::function_local_hashes; use crate::planner_identity::main::hook_version_hashes::hook_version_hashes; use crate::planner_identity::main::model_version_identity_metadata_json::model_version_identity_metadata_json; diff --git a/crates/sqlbuild-python-planner/src/consumption/_helpers/policy.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/policy.rs index 299520c88f..940a0e48ab 100644 --- a/crates/sqlbuild-python-planner/src/consumption/_helpers/policy.rs +++ b/crates/sqlbuild-python-planner/src/consumption/_helpers/policy.rs @@ -3,23 +3,18 @@ use std::collections::BTreeSet; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_state::consumption::models::{ - ChangeKind, ChangedPolicy, NodeKey, Reach, TriggeredJob, -}; +use sqlbuild_state::consumption::models::{ChangeKind, NodeKey, TriggeredJob}; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; -use sqlbuild_python_project::compiled_project::main::triggered_jobs::triggered_jobs; -use sqlbuild_python_project::graph::models::NativeProjectGraph; -const DEFINITION: &str = "definition"; -const SOURCE_DATA: &str = "source_data"; -const REACH_JOB: &str = "job"; -const REACH_ALL: &str = "all"; +pub(crate) const DEFINITION: &str = "definition"; +pub(crate) const SOURCE_DATA: &str = "source_data"; +pub(crate) const REACH_JOB: &str = "job"; +pub(crate) const REACH_ALL: &str = "all"; -fn change_kind(name: &str) -> PyResult { +pub(crate) fn change_kind(name: &str) -> PyResult { match name { DEFINITION => Ok(ChangeKind::Definition), SOURCE_DATA => Ok(ChangeKind::SourceData), @@ -29,23 +24,15 @@ fn change_kind(name: &str) -> PyResult { } } -/// A change kind as `[changes] counts` and `--changed=` spell it. -pub(crate) const fn kind_value(kind: ChangeKind) -> &'static str { - match kind { - ChangeKind::Definition => DEFINITION, - ChangeKind::SourceData => SOURCE_DATA, - } -} - /// The request's `--select` selectors. -fn selectors(request: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn selectors(request: &Bound<'_, PyAny>) -> PyResult> { Ok(native_field(request, "select")? .extract::>>()? .unwrap_or_default()) } /// A triggered job with its selected nodes as consumption keys. -fn triggered_job((name, keys): (String, BTreeSet)) -> TriggeredJob { +pub(crate) fn triggered_job((name, keys): (String, BTreeSet)) -> TriggeredJob { TriggeredJob { name, nodes: keys @@ -54,76 +41,3 @@ fn triggered_job((name, keys): (String, BTreeSet)) -> TriggeredJob { .collect(), } } - -/// A reach as `[changes] reach` and `--reach` spell it. -pub(crate) const fn reach_value(reach: &Reach) -> &'static str { - match reach { - Reach::Job(_) => REACH_JOB, - Reach::All => REACH_ALL, - } -} - -/// The policy of a `--changed` request, or None without it; triggered jobs resolve on `project`. -pub(crate) fn changed_policy( - project_dir: &Bound<'_, PyAny>, - request: &Bound<'_, PyAny>, - project: &Bound<'_, PyAny>, -) -> PyResult> { - if !native_field(request, "changed")?.is_truthy()? { - return Ok(None); - } - let configured = configured_changes(project_dir)?; - let counts: Vec = match native_field(request, "changed_counts")?.extract()? { - Some(counts) => counts, - None => configured - .counts - .unwrap_or_else(|| vec![DEFINITION.to_owned()]), - }; - let reach: String = match native_field(request, "reach")?.extract()? { - Some(reach) => reach, - None => configured.reach.unwrap_or_else(|| REACH_JOB.to_owned()), - }; - let counts: BTreeSet = counts - .iter() - .map(|name| change_kind(name)) - .collect::>()?; - let reach: Reach = match reach.as_str() { - REACH_ALL => Reach::All, - REACH_JOB => { - let graph: PyRef<'_, NativeProjectGraph> = - native_field(project, "lineage_graph")?.extract()?; - Reach::Job( - triggered_jobs(project.py(), graph.graph(), &selectors(request)?)? - .into_iter() - .map(triggered_job) - .collect(), - ) - } - other => { - return Err(PyValueError::new_err(format!( - "unknown reach '{other}': expected {REACH_JOB} or {REACH_ALL}" - ))); - } - }; - Ok(Some(ChangedPolicy { counts, reach })) -} - -/// One line saying what the policy counts and how far it reaches, as build and plan print it. -pub(crate) fn policy_line(policy: &ChangedPolicy) -> String { - let label = |kind: &ChangeKind| match kind { - ChangeKind::Definition => "definition", - ChangeKind::SourceData => "source data", - }; - let counted: Vec<&str> = policy.counts.iter().map(label).collect(); - let uncounted: Vec = [ChangeKind::Definition, ChangeKind::SourceData] - .iter() - .filter(|kind| !policy.counts.contains(kind)) - .map(|kind| format!(" ({} not counted)", label(kind))) - .collect(); - format!( - "Counting: {}{} · reach: {}", - counted.join(", "), - uncounted.concat(), - reach_value(&policy.reach) - ) -} diff --git a/crates/sqlbuild-python-planner/src/consumption/_helpers/recorder.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/recorder.rs index 255f24fb13..9afb032387 100644 --- a/crates/sqlbuild-python-planner/src/consumption/_helpers/recorder.rs +++ b/crates/sqlbuild-python-planner/src/consumption/_helpers/recorder.rs @@ -14,15 +14,12 @@ use sqlbuild_state::consumption::main::consumption_history::consumption_history; use sqlbuild_state::consumption::main::read_back_history::read_back_history; use sqlbuild_state::consumption::main::started_append::started_append; use sqlbuild_state::consumption::models::{ - ChangedOutcome, ChangedPolicy, ConsumptionAppend, ConsumptionHistory, ConsumptionRow, - ForcedRun, HeldBack, InputFact, NodeFacts, NodeKey, Reach, RelationKey, RunReason, + ChangedOutcome, ChangedPolicy, ConsumptionAppend, ConsumptionHistory, ConsumptionRow, HeldBack, + InputFact, NodeFacts, NodeKey, Reach, RelationKey, RunReason, }; use uuid::Uuid; -use crate::consumption::_helpers::facts::BuildFacts; -use crate::consumption::_helpers::facts::MODEL; -use crate::consumption::_helpers::facts::SEED; -use crate::consumption::_helpers::facts::build_facts; +use crate::consumption::_helpers::facts::{BuildFacts, MODEL, SEED, build_facts}; use sqlbuild_python_front::native_host::models::{ ConsumptionConnection, Location, NodeCompleteHook, }; @@ -167,23 +164,8 @@ pub(crate) fn prepare<'py>( }) } -/// What `plan --changed` previews: what a build would run now, read without recording anything. -pub(crate) fn preview_changed( - (plan, project): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - (policy, full_refresh): (&ChangedPolicy, bool), -) -> PyResult<(ChangedDecision, Vec)> { - let py: Python<'_> = plan.py(); - let facts: BuildFacts = build_facts((plan, project), (true, full_refresh))?; - let connection = ConsumptionConnection::new(adapter, connection_config); - let (history, warnings): (ConsumptionHistory, Vec) = - read_history(py, &connection, (&facts, Some(policy))); - connection.close(py)?; - Ok((changed_decision(&facts, &history, policy), warnings)) -} - /// Recent executions the build reads, with older ones read back when deciding needs them. -fn read_history( +pub(crate) fn read_history( py: Python<'_>, connection: &ConsumptionConnection, (facts, policy): (&BuildFacts, Option<&ChangedPolicy>), @@ -210,7 +192,7 @@ fn read_history( } /// What `--changed` does with each runnable node. -fn changed_decision( +pub(crate) fn changed_decision( facts: &BuildFacts, history: &ConsumptionHistory, policy: &ChangedPolicy, @@ -284,38 +266,3 @@ fn read_references( } (rows, warnings) } - -/// Why `--changed` holds a node back, as one line for verbose output. -pub(crate) fn held_back_text(why: &HeldBack) -> String { - match why { - HeldBack::RefreshedBy(jobs) => format!( - "job {} refreshes it, and the change is not in that job", - jobs.join(", ") - ), - HeldBack::ReadsHeldBack(key) => format!("reads held-back {} {}", key.kind, key.name), - } -} - -/// Why `--changed` runs a node, as one line for verbose output. -pub(crate) fn run_reason_text(reason: &RunReason) -> String { - match reason { - RunReason::Forced(ForcedRun::PlannerChange(kind)) => format!("planner change ({kind})"), - RunReason::Forced(ForcedRun::FullRefresh) => "full refresh".to_owned(), - RunReason::Forced(ForcedRun::RelationMissing) => "relation missing".to_owned(), - RunReason::Forced(ForcedRun::CursorInterval) => "cursor interval requested".to_owned(), - RunReason::Forced(ForcedRun::InputsNotEnumerable) => { - "inputs cannot be enumerated".to_owned() - } - RunReason::NoRecord => "no completed record".to_owned(), - RunReason::Unfinished => "last execution did not finish".to_owned(), - RunReason::AmbiguousOrder => "record order is ambiguous".to_owned(), - RunReason::DefinitionChanged => "definition changed".to_owned(), - RunReason::InputsChanged => "inputs changed".to_owned(), - RunReason::InputChanged(input) => format!("input {input} changed"), - RunReason::UpstreamDefinitionChanged(input) => { - format!("upstream {input} was rebuilt with a definition change") - } - RunReason::InputUnknown(input) => format!("input {input} has no known version"), - RunReason::UpstreamRuns(key) => format!("upstream {} {} runs", key.kind, key.name), - } -} diff --git a/crates/sqlbuild-python-planner/src/consumption/_helpers/relation_reads.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/relation_reads.rs index d5f910f5d1..25bb384cb0 100644 --- a/crates/sqlbuild-python-planner/src/consumption/_helpers/relation_reads.rs +++ b/crates/sqlbuild-python-planner/src/consumption/_helpers/relation_reads.rs @@ -23,15 +23,6 @@ pub(crate) struct Locations { by_name: HashMap>, } -/// The relation a model or seed destination names; a destination without a schema has an empty one. -pub(crate) fn relation(destination: &Bound<'_, PyAny>) -> PyResult { - Ok(RelationKey { - database: optional_text(&native_field(destination, "database")?)?, - schema: optional_text(&native_field(destination, "schema")?)?.unwrap_or_default(), - name: native_field(destination, "name")?.extract()?, - }) -} - impl Locations { /// `nodes` holds every model and seed relation of the project. pub(crate) fn read( diff --git a/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs b/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs index 93339cad0d..ca236b42d0 100644 --- a/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs +++ b/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs @@ -1,17 +1,61 @@ //! The policy of a `--changed` request, or None without it; triggered jobs resolve on `project`. -use crate::consumption::_helpers::policy; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use sqlbuild_state::consumption::models::ChangedPolicy; +use crate::consumption::_helpers::policy::DEFINITION; +use crate::consumption::_helpers::policy::REACH_ALL; +use crate::consumption::_helpers::policy::REACH_JOB; +use crate::consumption::_helpers::policy::change_kind; +use crate::consumption::_helpers::policy::selectors; +use crate::consumption::_helpers::policy::triggered_job; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; +use sqlbuild_python_project::compiled_project::main::triggered_jobs::triggered_jobs; +use sqlbuild_python_project::graph::models::NativeProjectGraph; +use sqlbuild_state::consumption::models::{ChangeKind, ChangedPolicy, Reach}; +use std::collections::BTreeSet; /// The policy of a `--changed` request, or None without it; triggered jobs resolve on `project`. -#[inline] pub fn changed_policy( project_dir: &Bound<'_, PyAny>, request: &Bound<'_, PyAny>, project: &Bound<'_, PyAny>, ) -> PyResult> { - policy::changed_policy(project_dir, request, project) + if !native_field(request, "changed")?.is_truthy()? { + return Ok(None); + } + let configured = configured_changes(project_dir)?; + let counts: Vec = match native_field(request, "changed_counts")?.extract()? { + Some(counts) => counts, + None => configured + .counts + .unwrap_or_else(|| vec![DEFINITION.to_owned()]), + }; + let reach: String = match native_field(request, "reach")?.extract()? { + Some(reach) => reach, + None => configured.reach.unwrap_or_else(|| REACH_JOB.to_owned()), + }; + let counts: BTreeSet = counts + .iter() + .map(|name| change_kind(name)) + .collect::>()?; + let reach: Reach = match reach.as_str() { + REACH_ALL => Reach::All, + REACH_JOB => { + let graph: PyRef<'_, NativeProjectGraph> = + native_field(project, "lineage_graph")?.extract()?; + Reach::Job( + triggered_jobs(project.py(), graph.graph(), &selectors(request)?)? + .into_iter() + .map(triggered_job) + .collect(), + ) + } + other => { + return Err(PyValueError::new_err(format!( + "unknown reach '{other}': expected {REACH_JOB} or {REACH_ALL}" + ))); + } + }; + Ok(Some(ChangedPolicy { counts, reach })) } diff --git a/crates/sqlbuild-python-planner/src/consumption/main/held_back_text.rs b/crates/sqlbuild-python-planner/src/consumption/main/held_back_text.rs index 8dd3dee28a..3b03a2bb96 100644 --- a/crates/sqlbuild-python-planner/src/consumption/main/held_back_text.rs +++ b/crates/sqlbuild-python-planner/src/consumption/main/held_back_text.rs @@ -1,10 +1,14 @@ //! Why `--changed` holds a node back, as one line for verbose output. -use crate::consumption::_helpers::recorder; use sqlbuild_state::consumption::models::HeldBack; /// Why `--changed` holds a node back, as one line for verbose output. -#[inline] pub fn held_back_text(why: &HeldBack) -> String { - recorder::held_back_text(why) + match why { + HeldBack::RefreshedBy(jobs) => format!( + "job {} refreshes it, and the change is not in that job", + jobs.join(", ") + ), + HeldBack::ReadsHeldBack(key) => format!("reads held-back {} {}", key.kind, key.name), + } } diff --git a/crates/sqlbuild-python-planner/src/consumption/main/kind_value.rs b/crates/sqlbuild-python-planner/src/consumption/main/kind_value.rs index a8e7e00e98..70d7c85ee5 100644 --- a/crates/sqlbuild-python-planner/src/consumption/main/kind_value.rs +++ b/crates/sqlbuild-python-planner/src/consumption/main/kind_value.rs @@ -1,10 +1,13 @@ //! A change kind as `[changes] counts` and `--changed=` spell it. -use crate::consumption::_helpers::policy; +use crate::consumption::_helpers::policy::DEFINITION; +use crate::consumption::_helpers::policy::SOURCE_DATA; use sqlbuild_state::consumption::models::ChangeKind; /// A change kind as `[changes] counts` and `--changed=` spell it. -#[inline] -pub fn kind_value(kind: ChangeKind) -> &'static str { - policy::kind_value(kind) +pub const fn kind_value(kind: ChangeKind) -> &'static str { + match kind { + ChangeKind::Definition => DEFINITION, + ChangeKind::SourceData => SOURCE_DATA, + } } diff --git a/crates/sqlbuild-python-planner/src/consumption/main/policy_line.rs b/crates/sqlbuild-python-planner/src/consumption/main/policy_line.rs index bcae7da848..bb5e3f5068 100644 --- a/crates/sqlbuild-python-planner/src/consumption/main/policy_line.rs +++ b/crates/sqlbuild-python-planner/src/consumption/main/policy_line.rs @@ -1,10 +1,24 @@ //! One line saying what the policy counts and how far it reaches, as build and plan print it. -use crate::consumption::_helpers::policy; -use sqlbuild_state::consumption::models::ChangedPolicy; +use crate::consumption::main::reach_value::reach_value; +use sqlbuild_state::consumption::models::{ChangeKind, ChangedPolicy}; /// One line saying what the policy counts and how far it reaches, as build and plan print it. -#[inline] pub fn policy_line(policy: &ChangedPolicy) -> String { - policy::policy_line(policy) + let label = |kind: &ChangeKind| match kind { + ChangeKind::Definition => "definition", + ChangeKind::SourceData => "source data", + }; + let counted: Vec<&str> = policy.counts.iter().map(label).collect(); + let uncounted: Vec = [ChangeKind::Definition, ChangeKind::SourceData] + .iter() + .filter(|kind| !policy.counts.contains(kind)) + .map(|kind| format!(" ({} not counted)", label(kind))) + .collect(); + format!( + "Counting: {}{} · reach: {}", + counted.join(", "), + uncounted.concat(), + reach_value(&policy.reach) + ) } diff --git a/crates/sqlbuild-python-planner/src/consumption/main/prepare.rs b/crates/sqlbuild-python-planner/src/consumption/main/prepare.rs index ad53731267..c80d70d8a4 100644 --- a/crates/sqlbuild-python-planner/src/consumption/main/prepare.rs +++ b/crates/sqlbuild-python-planner/src/consumption/main/prepare.rs @@ -2,9 +2,7 @@ use crate::consumption::_helpers::recorder; use crate::consumption::models::Prepared; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_state::consumption::models::ChangedPolicy; /// Read records, decide `--changed` skips, mint execution IDs and append `started` rows. diff --git a/crates/sqlbuild-python-planner/src/consumption/main/preview_changed.rs b/crates/sqlbuild-python-planner/src/consumption/main/preview_changed.rs index 256194047c..0f5cb9af37 100644 --- a/crates/sqlbuild-python-planner/src/consumption/main/preview_changed.rs +++ b/crates/sqlbuild-python-planner/src/consumption/main/preview_changed.rs @@ -1,22 +1,24 @@ //! What `plan --changed` previews: what a build would run now, read without recording anything. -use crate::consumption::_helpers::recorder; +use crate::consumption::_helpers::facts::{BuildFacts, build_facts}; +use crate::consumption::_helpers::recorder::changed_decision; +use crate::consumption::_helpers::recorder::read_history; use crate::consumption::models::ChangedDecision; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use sqlbuild_state::consumption::models::ChangedPolicy; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_front::native_host::models::ConsumptionConnection; +use sqlbuild_state::consumption::models::{ChangedPolicy, ConsumptionHistory}; /// What `plan --changed` previews: what a build would run now, read without recording anything. -#[inline] pub fn preview_changed( (plan, project): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), (policy, full_refresh): (&ChangedPolicy, bool), ) -> PyResult<(ChangedDecision, Vec)> { - recorder::preview_changed( - (plan, project), - (adapter, connection_config), - (policy, full_refresh), - ) + let py: Python<'_> = plan.py(); + let facts: BuildFacts = build_facts((plan, project), (true, full_refresh))?; + let connection = ConsumptionConnection::new(adapter, connection_config); + let (history, warnings): (ConsumptionHistory, Vec) = + read_history(py, &connection, (&facts, Some(policy))); + connection.close(py)?; + Ok((changed_decision(&facts, &history, policy), warnings)) } diff --git a/crates/sqlbuild-python-planner/src/consumption/main/reach_value.rs b/crates/sqlbuild-python-planner/src/consumption/main/reach_value.rs index cc13d319ff..e00f3477db 100644 --- a/crates/sqlbuild-python-planner/src/consumption/main/reach_value.rs +++ b/crates/sqlbuild-python-planner/src/consumption/main/reach_value.rs @@ -1,10 +1,13 @@ //! A reach as `[changes] reach` and `--reach` spell it. -use crate::consumption::_helpers::policy; +use crate::consumption::_helpers::policy::REACH_ALL; +use crate::consumption::_helpers::policy::REACH_JOB; use sqlbuild_state::consumption::models::Reach; /// A reach as `[changes] reach` and `--reach` spell it. -#[inline] -pub fn reach_value(reach: &Reach) -> &'static str { - policy::reach_value(reach) +pub const fn reach_value(reach: &Reach) -> &'static str { + match reach { + Reach::Job(_) => REACH_JOB, + Reach::All => REACH_ALL, + } } diff --git a/crates/sqlbuild-python-planner/src/consumption/main/relation.rs b/crates/sqlbuild-python-planner/src/consumption/main/relation.rs index 8fc9e4aceb..d3b1b84cdb 100644 --- a/crates/sqlbuild-python-planner/src/consumption/main/relation.rs +++ b/crates/sqlbuild-python-planner/src/consumption/main/relation.rs @@ -1,13 +1,15 @@ //! The relation a model or seed destination names; a destination without a schema has an empty one. -use crate::consumption::_helpers::relation_reads; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::native_host::main::optional_text::optional_text; use sqlbuild_state::consumption::models::RelationKey; /// The relation a model or seed destination names; a destination without a schema has an empty one. -#[inline] pub fn relation(destination: &Bound<'_, PyAny>) -> PyResult { - relation_reads::relation(destination) + Ok(RelationKey { + database: optional_text(&native_field(destination, "database")?)?, + schema: optional_text(&native_field(destination, "schema")?)?.unwrap_or_default(), + name: native_field(destination, "name")?.extract()?, + }) } diff --git a/crates/sqlbuild-python-planner/src/consumption/main/run_reason_text.rs b/crates/sqlbuild-python-planner/src/consumption/main/run_reason_text.rs index b6b2a62d8b..50d60da689 100644 --- a/crates/sqlbuild-python-planner/src/consumption/main/run_reason_text.rs +++ b/crates/sqlbuild-python-planner/src/consumption/main/run_reason_text.rs @@ -1,10 +1,27 @@ //! Why `--changed` runs a node, as one line for verbose output. -use crate::consumption::_helpers::recorder; -use sqlbuild_state::consumption::models::RunReason; +use sqlbuild_state::consumption::models::{ForcedRun, RunReason}; /// Why `--changed` runs a node, as one line for verbose output. -#[inline] pub fn run_reason_text(reason: &RunReason) -> String { - recorder::run_reason_text(reason) + match reason { + RunReason::Forced(ForcedRun::PlannerChange(kind)) => format!("planner change ({kind})"), + RunReason::Forced(ForcedRun::FullRefresh) => "full refresh".to_owned(), + RunReason::Forced(ForcedRun::RelationMissing) => "relation missing".to_owned(), + RunReason::Forced(ForcedRun::CursorInterval) => "cursor interval requested".to_owned(), + RunReason::Forced(ForcedRun::InputsNotEnumerable) => { + "inputs cannot be enumerated".to_owned() + } + RunReason::NoRecord => "no completed record".to_owned(), + RunReason::Unfinished => "last execution did not finish".to_owned(), + RunReason::AmbiguousOrder => "record order is ambiguous".to_owned(), + RunReason::DefinitionChanged => "definition changed".to_owned(), + RunReason::InputsChanged => "inputs changed".to_owned(), + RunReason::InputChanged(input) => format!("input {input} changed"), + RunReason::UpstreamDefinitionChanged(input) => { + format!("upstream {input} was rebuilt with a definition change") + } + RunReason::InputUnknown(input) => format!("input {input} has no known version"), + RunReason::UpstreamRuns(key) => format!("upstream {} {} runs", key.kind, key.name), + } } diff --git a/crates/sqlbuild-python-planner/src/freshness/_helpers/exception_types.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/exception_types.rs index 8d6c959c03..0ddcd1b496 100644 --- a/crates/sqlbuild-python-planner/src/freshness/_helpers/exception_types.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/exception_types.rs @@ -2,7 +2,6 @@ use pyo3::create_exception; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; create_exception!( sqlbuild._native, @@ -16,15 +15,3 @@ create_exception!( PyValueError, "Raised when source freshness cannot be observed safely." ); - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - let py = module.py(); - module.add( - "SourceFreshnessInputError", - py.get_type::(), - )?; - module.add( - "SourceFreshnessObservationError", - py.get_type::(), - ) -} diff --git a/crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs index 9be12b1ec2..37f08a5d38 100644 --- a/crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs @@ -4,20 +4,16 @@ use crate::freshness::models::PreviousState; use std::collections::{HashMap, HashSet}; use pyo3::exceptions::{PyException, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::freshness_text::main::freshness_json::freshness_json; use sqlbuild_host::entries::models::ExecutorHost; -use crate::freshness::_helpers::observation::observation_record; -use crate::freshness::_helpers::observation::observation_source; -use crate::freshness::_helpers::observation::observe_adapter_batch; -use crate::freshness::_helpers::observation::observe_configured; -use crate::freshness::_helpers::observation::strategy; +use crate::freshness::_helpers::observation::{ + observation_record, observation_source, observe_adapter_batch, observe_configured, strategy, +}; use crate::freshness::models::Observer; use sqlbuild_core::freshness_text::main::freshness_status::freshness_status; use sqlbuild_core::freshness_text::models::{ @@ -298,7 +294,7 @@ pub(crate) fn observe_freshness( /// Observe source freshness and return the `sqb freshness --json` payload. #[pyfunction] #[pyo3(signature = (**options))] -fn observe_source_freshness_json( +pub(crate) fn observe_source_freshness_json( py: Python<'_>, options: Option>, ) -> PyResult { @@ -338,8 +334,3 @@ fn observe_source_freshness_json( )?; freshness_json(&results).map_err(|error| PyValueError::new_err(format!("{error:?}"))) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(observe_source_freshness_json, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs index e3ebc77ef1..cc734a6bcf 100644 --- a/crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs @@ -1,24 +1,20 @@ //! Observe configured and adapter-metadata source freshness through adapter callbacks. +use crate::freshness::main::presumed_utc_datetime::presumed_utc_datetime; use crate::freshness::models::Observer; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDateTime, PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use sqlbuild_state::freshness::main::single_value_problem::single_value_problem; use sqlbuild_state::freshness::models::QueryShape; -use crate::freshness::_helpers::support::OBSERVATION_ERROR; -use crate::freshness::_helpers::support::exception_text; -use crate::freshness::_helpers::support::freshness_error; -use crate::freshness::_helpers::support::presumed_utc_datetime; -use crate::freshness::_helpers::support::presumed_utc_value; -use crate::freshness::_helpers::support::text; +use crate::freshness::_helpers::support::{ + OBSERVATION_ERROR, exception_text, freshness_error, presumed_utc_value, text, +}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::constants::DATA_VERSION_FIELD; -use sqlbuild_python_base::records::constants::OBSERVED_AT_FIELD; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -27,9 +23,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_front::state_rows::main::event_rows::in_utc::in_utc; use sqlbuild_python_front::state_rows::main::event_rows::normalize::normalize; -use sqlbuild_python_front::state_rows::main::event_rows::offset_aware::offset_aware; use sqlbuild_python_front::state_rows::main::event_rows::version_hash::version_hash; use sqlbuild_python_project::references::main::render_source_read::render_source_read; @@ -39,7 +33,7 @@ const DEFAULT_STRATEGY: &str = "sql"; const METADATA_OPERATION: &str = "source_freshness_metadata_observation"; const QUERY_OPERATION: &str = "source_freshness_query_observation"; -fn observation_error(py: Python<'_>, message: &str) -> pyo3::PyErr { +pub(crate) fn observation_error(py: Python<'_>, message: &str) -> pyo3::PyErr { freshness_error(py, (OBSERVATION_ERROR, message), None) } @@ -555,7 +549,7 @@ pub(crate) fn observation_record<'py>( /// Observe one source freshness config and return a comparable data version. #[pyfunction] #[pyo3(signature = (*, adapter, connection, source, observed_at))] -fn observe_configured_source_freshness<'py>( +pub(crate) fn observe_configured_source_freshness<'py>( adapter: Bound<'py, PyAny>, connection: Bound<'py, PyAny>, source: &Bound<'py, PyAny>, @@ -572,7 +566,7 @@ fn observe_configured_source_freshness<'py>( /// Observe adapter metadata freshness per source, batching the warehouse lookups. #[pyfunction] #[pyo3(signature = (*, adapter, connection, sources, observed_at))] -fn observe_adapter_sources_freshness<'py>( +pub(crate) fn observe_adapter_sources_freshness<'py>( adapter: Bound<'py, PyAny>, connection: Bound<'py, PyAny>, sources: &Bound<'py, PyAny>, @@ -586,43 +580,3 @@ fn observe_adapter_sources_freshness<'py>( let sources: Vec> = sources.try_iter()?.collect::>()?; observe_adapter_batch(&observer, &sources) } - -/// An aware freshness `datetime` in UTC; a naive value is an observation error. -pub(crate) fn require_aware_utc_datetime<'py>( - value: &Bound<'py, PyAny>, - field_name: &str, -) -> PyResult> { - if !offset_aware(value)? { - return Err(observation_error( - value.py(), - &format!("source freshness {field_name} must be timezone-aware"), - )); - } - in_utc(value) -} - -/// Hold an observation's `observed_at` and datetime `data_version` as aware UTC values. -pub(crate) fn normalize_observation( - py: Python<'_>, - mut values: Vec>, -) -> PyResult>> { - let observed_at: Bound<'_, PyAny> = - require_aware_utc_datetime(values[OBSERVED_AT_FIELD].bind(py), "observed_at")?; - values[OBSERVED_AT_FIELD] = observed_at.unbind(); - let data_version: &Bound<'_, PyAny> = values[DATA_VERSION_FIELD].bind(py); - if data_version.is_instance_of::() { - let data_version: Bound<'_, PyAny> = - require_aware_utc_datetime(data_version, "data_version")?; - values[DATA_VERSION_FIELD] = data_version.unbind(); - } - Ok(values) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - observe_configured_source_freshness, - module - )?)?; - module.add_function(wrap_pyfunction!(observe_adapter_sources_freshness, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs index 9c29d4f93d..4829e50f12 100644 --- a/crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs @@ -3,11 +3,9 @@ use crate::freshness::constants::PREVIEW_MESSAGE; use std::collections::{HashMap, HashSet}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::planner_identity::constants::{MODEL_RESOURCE, SOURCE_RESOURCE}; use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; @@ -15,12 +13,10 @@ use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_state::freshness::main::configuration_error::configuration_error; use crate::freshness::_helpers::exception_types::SourceFreshnessObservationError; -use crate::freshness::_helpers::observation::observation_record; -use crate::freshness::_helpers::observation::observation_source; -use crate::freshness::_helpers::observation::observe_adapter_batch; -use crate::freshness::_helpers::observation::observe_configured; -use crate::freshness::_helpers::observation::strategy; -use crate::freshness::_helpers::observation::unknown; +use crate::freshness::_helpers::observation::{ + observation_record, observation_source, observe_adapter_batch, observe_configured, strategy, + unknown, +}; use crate::freshness::_helpers::state_io::read_latest; use crate::freshness::_helpers::support::text; use crate::freshness::models::Observer; @@ -394,15 +390,3 @@ pub(crate) fn build_direct_source_freshness_propagation_result<'py>( ], ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - build_direct_source_freshness_planning_result, - module - )?)?; - module.add_function(wrap_pyfunction!( - build_direct_source_freshness_propagation_result, - module - )?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-planner/src/freshness/_helpers/publication.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/publication.rs index 0c72f2cab2..b43220bb06 100644 --- a/crates/sqlbuild-python-planner/src/freshness/_helpers/publication.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/publication.rs @@ -1,12 +1,12 @@ //! Append observed source freshness to state once the models it affected have built. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::freshness::main::append_freshness_records::append_freshness_records; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; use crate::freshness::_helpers::state_io::write_source_freshness_records; -use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; @@ -20,7 +20,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cost_context::models::CostContext; /// Observed records grouped by `(database, schema)` in first-seen order. -type RecordsByLocation<'py> = Vec<( +pub(crate) type RecordsByLocation<'py> = Vec<( (Bound<'py, PyAny>, Bound<'py, PyAny>), Vec>, )>; @@ -103,7 +103,7 @@ fn contains_location( } /// The observed records to publish, keyed by the schema of each affected model. -fn eligible_records<'py>( +pub(crate) fn eligible_records<'py>( plan: &Bound<'py, PyAny>, successful_models: &[String], run_id: &Bound<'py, PyAny>, @@ -188,7 +188,7 @@ fn renderers<'py>(adapter: &Bound<'py, PyAny>) -> PyResult> { } /// Write each location's records, one cost scope per source. -fn write_records<'py>( +pub(crate) fn write_records<'py>( adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, grouped: &RecordsByLocation<'py>, @@ -250,58 +250,10 @@ fn write_records<'py>( Ok(()) } -/// Append observed source freshness only after the affected selected models succeed. -pub(crate) fn append_freshness_records<'py>( - (plan, successful_models): (&Bound<'py, PyAny>, &[String]), - adapter: &Bound<'py, PyAny>, - connection_config: &Bound<'py, PyAny>, - run_id: &Bound<'py, PyAny>, -) -> PyResult<()> { - let py: Python<'py> = plan.py(); - let grouped: RecordsByLocation<'py> = eligible_records(plan, successful_models, run_id)?; - let record_count: usize = grouped.iter().map(|(_, records)| records.len()).sum(); - if record_count == 0 { - return Ok(()); - } - let run_scope: Bound<'py, PyAny> = host_entry(py, ExecutorHost::RunScope)?.call1((run_id,))?; - within(&run_scope, |_| { - let metadata: Bound<'py, PyDict> = PyDict::new(py); - metadata.set_item("item_count", record_count)?; - let attributes: Bound<'py, PyAny> = call_with( - &host_entry(py, ExecutorHost::OperationAttributes)?, - &[ - ("phase", PyString::new(py, "publish").into_any()), - ("target_kind", PyString::new(py, "state_batch").into_any()), - ], - )?; - let lifecycle: Bound<'py, PyAny> = call_with( - &host_entry(py, ExecutorHost::OperationLifecycle)?, - &[ - ("operation_kind", PyString::new(py, "freshness").into_any()), - ( - "operation_name", - PyString::new(py, "source_freshness_publication").into_any(), - ), - ("metadata", metadata.into_any()), - ("attributes", attributes), - ], - )?; - within(&lifecycle, |_| { - let connection: Bound<'py, PyAny> = - adapter.call_method1("connect", (connection_config,))?; - let written: PyResult<()> = write_records(adapter, &connection, &grouped); - let closed: PyResult> = adapter.call_method1("close", (&connection,)); - written?; - closed?; - Ok(()) - }) - }) -} - /// Append observed source freshness only after the affected selected models succeed. #[pyfunction] #[pyo3(signature = (**options))] -fn append_eligible_direct_source_freshness_records<'py>( +pub(crate) fn append_eligible_direct_source_freshness_records<'py>( py: Python<'py>, options: Option>, ) -> PyResult<()> { @@ -324,10 +276,3 @@ fn append_eligible_direct_source_freshness_records<'py>( &arguments.value("run_id")?, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - append_eligible_direct_source_freshness_records, - module - )?) -} diff --git a/crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs index 661bb0bdde..6bcbeed0ce 100644 --- a/crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs @@ -1,9 +1,10 @@ //! Source freshness state table DDL, latest-row reads and appends through adapter callbacks. +use crate::freshness::main::row_record::row_record; use crate::freshness::models::StateRead; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; use sqlbuild_state::freshness::constants::{ REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_COLUMN_TYPES, SOURCE_FRESHNESS_EVENT_ID, @@ -12,22 +13,14 @@ use sqlbuild_state::freshness::constants::{ use sqlbuild_state::freshness::main::read_latest_sql::read_latest_sql; use sqlbuild_state::freshness::main::relevant_sources_sql::relevant_sources_sql; -use crate::freshness::_helpers::support::INPUT_ERROR; -use crate::freshness::_helpers::support::exception_text; -use crate::freshness::_helpers::support::freshness_error; -use crate::freshness::_helpers::support::optional_str; -use crate::freshness::_helpers::support::presumed_utc_datetime; -use crate::freshness::_helpers::support::text; +use crate::freshness::_helpers::support::{INPUT_ERROR, exception_text, freshness_error, text}; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::main::ensure_event_id_column::ensure_event_id_column; -use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; use sqlbuild_python_front::native_host::main::state_table_name::state_table_name; -use sqlbuild_python_front::state_rows::main::event_rows::date_time_typed::date_time_typed; use sqlbuild_python_front::state_rows::main::tables::state_table_create_sql::state_table_create_sql; const MISSING_SCHEMA_MESSAGE: &str = "source freshness table requires a target schema"; @@ -51,7 +44,7 @@ pub(crate) fn qualified_table_name( /// SQL selecting the latest source freshness row per identity. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name))] -fn build_source_freshness_read_latest_sql( +pub(crate) fn build_source_freshness_read_latest_sql( database: &Bound<'_, PyAny>, schema: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, @@ -65,7 +58,7 @@ fn build_source_freshness_read_latest_sql( /// Create-if-missing DDL for the source freshness state table. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name, render_framework_type))] -fn build_source_freshness_create_table_sql<'py>( +pub(crate) fn build_source_freshness_create_table_sql<'py>( database: &Bound<'py, PyAny>, schema: &Bound<'py, PyAny>, render_qualified_name: &Bound<'py, PyAny>, @@ -183,43 +176,10 @@ freshness table to regenerate source freshness state." ) } -pub(crate) fn row_record<'py>(row: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = row.py(); - let raw_observed_at: Bound<'py, PyAny> = row.get_item(9)?; - let observed_at: Bound<'py, PyAny> = if date_time_typed(&raw_observed_at)? { - raw_observed_at - } else { - imported(py, DATETIME_MODULE, "datetime")? - .call_method1("fromisoformat", (raw_observed_at.str()?,))? - }; - let observed_at: Bound<'py, PyAny> = presumed_utc_datetime(&observed_at)?; - let field = |index: usize| -> PyResult> { - Ok(row.get_item(index)?.str()?.into_any()) - }; - let optional = - |index: usize| -> PyResult> { optional_str(&row.get_item(index)?) }; - new_record_with( - py, - RecordId::SourceFreshnessRecord, - &[ - ("source_name", field(0)?), - ("target_database", optional(1)?), - ("target_schema", optional(2)?), - ("target_name", optional(3)?), - ("run_id", field(4)?), - ("strategy", field(5)?), - ("value_kind", field(6)?), - ("data_version", optional(7)?), - ("data_version_hash", field(8)?), - ("observed_at", observed_at), - ], - ) -} - /// Read latest source freshness rows, trusting the caller-resolved table existence. #[pyfunction] #[pyo3(signature = (**options))] -fn read_latest_source_freshness<'py>( +pub(crate) fn read_latest_source_freshness<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -337,17 +297,3 @@ pub(crate) fn write_source_freshness_records<'py>( )?; execute_sql((execute, connection), insert_sql) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - build_source_freshness_read_latest_sql, - module - )?)?; - module.add_function(wrap_pyfunction!( - build_source_freshness_create_table_sql, - module - )?)?; - module.add_function(wrap_pyfunction!(read_latest_source_freshness, module)?)?; - module.add_function(wrap_pyfunction!(write_source_freshness_records, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs index dac1690a74..f4b87f48bf 100644 --- a/crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs @@ -1,16 +1,14 @@ //! Python objects, exceptions and datetimes the source freshness glue shares. +use crate::freshness::main::presumed_utc_datetime::presumed_utc_datetime; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyString}; +use pyo3::types::PyString; -use crate::freshness::_helpers::exception_types::SourceFreshnessInputError; -use crate::freshness::_helpers::exception_types::SourceFreshnessObservationError; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; +use crate::freshness::_helpers::exception_types::{ + SourceFreshnessInputError, SourceFreshnessObservationError, +}; use sqlbuild_python_front::state_rows::main::event_rows::date_time_typed::date_time_typed; -use sqlbuild_python_front::state_rows::main::event_rows::in_utc::in_utc; -use sqlbuild_python_front::state_rows::main::event_rows::offset_aware::offset_aware; pub(crate) const OBSERVATION_ERROR: fn(String) -> PyErr = SourceFreshnessObservationError::new_err; pub(crate) const INPUT_ERROR: fn(String) -> PyErr = SourceFreshnessInputError::new_err; @@ -44,16 +42,6 @@ pub(crate) fn presumed_utc_value<'py>(value: &Bound<'py, PyAny>) -> PyResult(value: &Bound<'py, PyAny>) -> PyResult> { - if offset_aware(value)? { - return in_utc(value); - } - let changes: Bound<'py, PyDict> = PyDict::new(value.py()); - changes.set_item("tzinfo", imported(value.py(), DATETIME_MODULE, "UTC")?)?; - value.call_method("replace", (), Some(&changes)) -} - pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() } diff --git a/crates/sqlbuild-python-planner/src/freshness/main/append_freshness_records.rs b/crates/sqlbuild-python-planner/src/freshness/main/append_freshness_records.rs index eddd32404a..8f1323b136 100644 --- a/crates/sqlbuild-python-planner/src/freshness/main/append_freshness_records.rs +++ b/crates/sqlbuild-python-planner/src/freshness/main/append_freshness_records.rs @@ -1,22 +1,59 @@ //! Append observed source freshness only after the affected selected models succeed. -use crate::freshness::_helpers::publication; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::freshness::_helpers::publication::RecordsByLocation; +use crate::freshness::_helpers::publication::eligible_records; +use crate::freshness::_helpers::publication::write_records; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; /// Append observed source freshness only after the affected selected models succeed. -#[inline] pub fn append_freshness_records<'py>( (plan, successful_models): (&Bound<'py, PyAny>, &[String]), adapter: &Bound<'py, PyAny>, connection_config: &Bound<'py, PyAny>, run_id: &Bound<'py, PyAny>, ) -> PyResult<()> { - publication::append_freshness_records( - (plan, successful_models), - adapter, - connection_config, - run_id, - ) + let py: Python<'py> = plan.py(); + let grouped: RecordsByLocation<'py> = eligible_records(plan, successful_models, run_id)?; + let record_count: usize = grouped.iter().map(|(_, records)| records.len()).sum(); + if record_count == 0 { + return Ok(()); + } + let run_scope: Bound<'py, PyAny> = host_entry(py, ExecutorHost::RunScope)?.call1((run_id,))?; + within(&run_scope, |_| { + let metadata: Bound<'py, PyDict> = PyDict::new(py); + metadata.set_item("item_count", record_count)?; + let attributes: Bound<'py, PyAny> = call_with( + &host_entry(py, ExecutorHost::OperationAttributes)?, + &[ + ("phase", PyString::new(py, "publish").into_any()), + ("target_kind", PyString::new(py, "state_batch").into_any()), + ], + )?; + let lifecycle: Bound<'py, PyAny> = call_with( + &host_entry(py, ExecutorHost::OperationLifecycle)?, + &[ + ("operation_kind", PyString::new(py, "freshness").into_any()), + ( + "operation_name", + PyString::new(py, "source_freshness_publication").into_any(), + ), + ("metadata", metadata.into_any()), + ("attributes", attributes), + ], + )?; + within(&lifecycle, |_| { + let connection: Bound<'py, PyAny> = + adapter.call_method1("connect", (connection_config,))?; + let written: PyResult<()> = write_records(adapter, &connection, &grouped); + let closed: PyResult> = adapter.call_method1("close", (&connection,)); + written?; + closed?; + Ok(()) + }) + }) } diff --git a/crates/sqlbuild-python-planner/src/freshness/main/normalize_observation.rs b/crates/sqlbuild-python-planner/src/freshness/main/normalize_observation.rs index e06c17d3f3..15d199d999 100644 --- a/crates/sqlbuild-python-planner/src/freshness/main/normalize_observation.rs +++ b/crates/sqlbuild-python-planner/src/freshness/main/normalize_observation.rs @@ -1,13 +1,24 @@ //! Hold an observation's `observed_at` and datetime `data_version` as aware UTC values. -use crate::freshness::_helpers::observation; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::freshness::main::require_aware_utc_datetime::require_aware_utc_datetime; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDateTime; +use sqlbuild_python_base::records::constants::DATA_VERSION_FIELD; +use sqlbuild_python_base::records::constants::OBSERVED_AT_FIELD; /// Hold an observation's `observed_at` and datetime `data_version` as aware UTC values. -#[inline] -pub fn normalize_observation(py: Python<'_>, values: Vec>) -> PyResult>> { - observation::normalize_observation(py, values) +pub fn normalize_observation( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + let observed_at: Bound<'_, PyAny> = + require_aware_utc_datetime(values[OBSERVED_AT_FIELD].bind(py), "observed_at")?; + values[OBSERVED_AT_FIELD] = observed_at.unbind(); + let data_version: &Bound<'_, PyAny> = values[DATA_VERSION_FIELD].bind(py); + if data_version.is_instance_of::() { + let data_version: Bound<'_, PyAny> = + require_aware_utc_datetime(data_version, "data_version")?; + values[DATA_VERSION_FIELD] = data_version.unbind(); + } + Ok(values) } diff --git a/crates/sqlbuild-python-planner/src/freshness/main/observe_freshness.rs b/crates/sqlbuild-python-planner/src/freshness/main/observe_freshness.rs index fee613d9f3..1732f96b79 100644 --- a/crates/sqlbuild-python-planner/src/freshness/main/observe_freshness.rs +++ b/crates/sqlbuild-python-planner/src/freshness/main/observe_freshness.rs @@ -3,9 +3,7 @@ use crate::freshness::_helpers::freshness_observe; use crate::freshness::models::Observer; use crate::freshness::models::PreviousState; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_core::freshness_text::models::FreshnessSourceResult; /// Observe the selected sources inside the run's observability scope. diff --git a/crates/sqlbuild-python-planner/src/freshness/main/presumed_utc_datetime.rs b/crates/sqlbuild-python-planner/src/freshness/main/presumed_utc_datetime.rs index 082387f417..98f2ecd68e 100644 --- a/crates/sqlbuild-python-planner/src/freshness/main/presumed_utc_datetime.rs +++ b/crates/sqlbuild-python-planner/src/freshness/main/presumed_utc_datetime.rs @@ -1,12 +1,18 @@ //! A `datetime` in UTC, a naive or offset-less value read as UTC. -use crate::freshness::_helpers::support; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; +use sqlbuild_python_front::state_rows::main::event_rows::in_utc::in_utc; +use sqlbuild_python_front::state_rows::main::event_rows::offset_aware::offset_aware; /// A `datetime` in UTC, a naive or offset-less value read as UTC. -#[inline] pub fn presumed_utc_datetime<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - support::presumed_utc_datetime(value) + if offset_aware(value)? { + return in_utc(value); + } + let changes: Bound<'py, PyDict> = PyDict::new(value.py()); + changes.set_item("tzinfo", imported(value.py(), DATETIME_MODULE, "UTC")?)?; + value.call_method("replace", (), Some(&changes)) } diff --git a/crates/sqlbuild-python-planner/src/freshness/main/read_latest.rs b/crates/sqlbuild-python-planner/src/freshness/main/read_latest.rs index e2f079ba04..c1f899cf88 100644 --- a/crates/sqlbuild-python-planner/src/freshness/main/read_latest.rs +++ b/crates/sqlbuild-python-planner/src/freshness/main/read_latest.rs @@ -2,9 +2,7 @@ use crate::freshness::_helpers::state_io; use crate::freshness::models::StateRead; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Latest records of one schema, trusting the caller-resolved table existence. #[inline] diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_exception_types.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_exception_types.rs index e6c1a66011..f3aa7b9dce 100644 --- a/crates/sqlbuild-python-planner/src/freshness/main/register_exception_types.rs +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_exception_types.rs @@ -1,12 +1,18 @@ //! Register the exception types functions and classes. -use crate::freshness::_helpers::exception_types; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::freshness::_helpers::exception_types::SourceFreshnessInputError; +use crate::freshness::_helpers::exception_types::SourceFreshnessObservationError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the exception types functions and classes. -#[inline] pub fn register_exception_types(module: &Bound<'_, PyModule>) -> PyResult<()> { - exception_types::register(module) + let py = module.py(); + module.add( + "SourceFreshnessInputError", + py.get_type::(), + )?; + module.add( + "SourceFreshnessObservationError", + py.get_type::(), + ) } diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_freshness_observe.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_freshness_observe.rs index 9a1da01e1b..fcc9d6671d 100644 --- a/crates/sqlbuild-python-planner/src/freshness/main/register_freshness_observe.rs +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_freshness_observe.rs @@ -1,12 +1,11 @@ //! Register the freshness observe functions and classes. -use crate::freshness::_helpers::freshness_observe; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::freshness::_helpers::freshness_observe::observe_source_freshness_json; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the freshness observe functions and classes. -#[inline] pub fn register_freshness_observe(module: &Bound<'_, PyModule>) -> PyResult<()> { - freshness_observe::register(module) + module.add_function(wrap_pyfunction!(observe_source_freshness_json, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_observation.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_observation.rs index 542b47c211..291cd0eef0 100644 --- a/crates/sqlbuild-python-planner/src/freshness/main/register_observation.rs +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_observation.rs @@ -1,12 +1,16 @@ //! Register the observation functions and classes. -use crate::freshness::_helpers::observation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::freshness::_helpers::observation::observe_adapter_sources_freshness; +use crate::freshness::_helpers::observation::observe_configured_source_freshness; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the observation functions and classes. -#[inline] pub fn register_observation(module: &Bound<'_, PyModule>) -> PyResult<()> { - observation::register(module) + module.add_function(wrap_pyfunction!( + observe_configured_source_freshness, + module + )?)?; + module.add_function(wrap_pyfunction!(observe_adapter_sources_freshness, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_planning.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_planning.rs index 5e8252fbbc..a1563d226a 100644 --- a/crates/sqlbuild-python-planner/src/freshness/main/register_planning.rs +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_planning.rs @@ -1,12 +1,19 @@ //! Register the planning functions and classes. -use crate::freshness::_helpers::planning; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::freshness::_helpers::planning::build_direct_source_freshness_planning_result; +use crate::freshness::_helpers::planning::build_direct_source_freshness_propagation_result; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the planning functions and classes. -#[inline] pub fn register_planning(module: &Bound<'_, PyModule>) -> PyResult<()> { - planning::register(module) + module.add_function(wrap_pyfunction!( + build_direct_source_freshness_planning_result, + module + )?)?; + module.add_function(wrap_pyfunction!( + build_direct_source_freshness_propagation_result, + module + )?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_publication.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_publication.rs index 0f2be945b7..ab766f21cf 100644 --- a/crates/sqlbuild-python-planner/src/freshness/main/register_publication.rs +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_publication.rs @@ -1,12 +1,13 @@ //! Register the publication functions and classes. -use crate::freshness::_helpers::publication; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::freshness::_helpers::publication::append_eligible_direct_source_freshness_records; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the publication functions and classes. -#[inline] pub fn register_publication(module: &Bound<'_, PyModule>) -> PyResult<()> { - publication::register(module) + module.add_function(wrap_pyfunction!( + append_eligible_direct_source_freshness_records, + module + )?) } diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs index f9170c48cb..b3ac616430 100644 --- a/crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs @@ -1,12 +1,23 @@ //! Register the state io functions and classes. -use crate::freshness::_helpers::state_io; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::freshness::_helpers::state_io::build_source_freshness_create_table_sql; +use crate::freshness::_helpers::state_io::build_source_freshness_read_latest_sql; +use crate::freshness::_helpers::state_io::read_latest_source_freshness; +use crate::freshness::_helpers::state_io::write_source_freshness_records; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the state io functions and classes. -#[inline] pub fn register_state_io(module: &Bound<'_, PyModule>) -> PyResult<()> { - state_io::register(module) + module.add_function(wrap_pyfunction!( + build_source_freshness_read_latest_sql, + module + )?)?; + module.add_function(wrap_pyfunction!( + build_source_freshness_create_table_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(read_latest_source_freshness, module)?)?; + module.add_function(wrap_pyfunction!(write_source_freshness_records, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-planner/src/freshness/main/require_aware_utc_datetime.rs b/crates/sqlbuild-python-planner/src/freshness/main/require_aware_utc_datetime.rs index d8c2df1d69..376406fbac 100644 --- a/crates/sqlbuild-python-planner/src/freshness/main/require_aware_utc_datetime.rs +++ b/crates/sqlbuild-python-planner/src/freshness/main/require_aware_utc_datetime.rs @@ -1,15 +1,20 @@ //! An aware freshness `datetime` in UTC; a naive value is an observation error. -use crate::freshness::_helpers::observation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::freshness::_helpers::observation::observation_error; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_front::state_rows::main::event_rows::in_utc::in_utc; +use sqlbuild_python_front::state_rows::main::event_rows::offset_aware::offset_aware; /// An aware freshness `datetime` in UTC; a naive value is an observation error. -#[inline] pub fn require_aware_utc_datetime<'py>( value: &Bound<'py, PyAny>, field_name: &str, ) -> PyResult> { - observation::require_aware_utc_datetime(value, field_name) + if !offset_aware(value)? { + return Err(observation_error( + value.py(), + &format!("source freshness {field_name} must be timezone-aware"), + )); + } + in_utc(value) } diff --git a/crates/sqlbuild-python-planner/src/freshness/main/row_record.rs b/crates/sqlbuild-python-planner/src/freshness/main/row_record.rs index 829f08045b..c96bd0218c 100644 --- a/crates/sqlbuild-python-planner/src/freshness/main/row_record.rs +++ b/crates/sqlbuild-python-planner/src/freshness/main/row_record.rs @@ -1,12 +1,44 @@ //! Row record. -use crate::freshness::_helpers::state_io; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::freshness::_helpers::support::optional_str; +use crate::freshness::main::presumed_utc_datetime::presumed_utc_datetime; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; +use sqlbuild_python_front::state_rows::main::event_rows::date_time_typed::date_time_typed; /// Row record. -#[inline] pub fn row_record<'py>(row: &Bound<'py, PyAny>) -> PyResult> { - state_io::row_record(row) + let py: Python<'py> = row.py(); + let raw_observed_at: Bound<'py, PyAny> = row.get_item(9)?; + let observed_at: Bound<'py, PyAny> = if date_time_typed(&raw_observed_at)? { + raw_observed_at + } else { + imported(py, DATETIME_MODULE, "datetime")? + .call_method1("fromisoformat", (raw_observed_at.str()?,))? + }; + let observed_at: Bound<'py, PyAny> = presumed_utc_datetime(&observed_at)?; + let field = |index: usize| -> PyResult> { + Ok(row.get_item(index)?.str()?.into_any()) + }; + let optional = + |index: usize| -> PyResult> { optional_str(&row.get_item(index)?) }; + new_record_with( + py, + RecordId::SourceFreshnessRecord, + &[ + ("source_name", field(0)?), + ("target_database", optional(1)?), + ("target_schema", optional(2)?), + ("target_name", optional(3)?), + ("run_id", field(4)?), + ("strategy", field(5)?), + ("value_kind", field(6)?), + ("data_version", optional(7)?), + ("data_version_hash", field(8)?), + ("observed_at", observed_at), + ], + ) } diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/catalog.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/catalog.rs index 411f03bc91..daa4233c91 100644 --- a/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/catalog.rs +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/catalog.rs @@ -1,15 +1,9 @@ //! The native lifecycle event catalogue: known types, terminality and catalogued values. use pyo3::exceptions::PyKeyError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyFrozenSet, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::lifecycle_events::constants::{ - CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, HOOK_PHASES, - HOOK_TYPES, LIFECYCLE_EVENT_SCHEMA_VERSIONS, OPERATION_ADAPTERS, OPERATION_KINDS, - OPERATION_METADATA_FIELDS, OPERATION_NAMES, OPERATION_PHASES, OPERATION_SCOPES, - OPERATION_STRATEGIES, OPERATION_TARGET_KINDS, RESOURCE_SKIP_CODES, RESOURCE_SKIP_MODES, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyString; use sqlbuild_core::lifecycle_events::main::catalog::catalog; use sqlbuild_core::lifecycle_events::models::EventDefinition; @@ -32,7 +26,7 @@ pub(crate) fn definition( /// Whether `event_type` is catalogued for `schema_version`. #[pyfunction] #[pyo3(signature = (*, schema_version, event_type))] -fn lifecycle_event_catalogued( +pub(crate) fn lifecycle_event_catalogued( schema_version: &Bound<'_, PyAny>, event_type: &Bound<'_, PyAny>, ) -> PyResult { @@ -42,7 +36,7 @@ fn lifecycle_event_catalogued( /// Whether a catalogued event closes its correlated lifecycle scope; `KeyError` otherwise. #[pyfunction] #[pyo3(signature = (*, schema_version, event_type))] -fn lifecycle_event_terminal( +pub(crate) fn lifecycle_event_terminal( schema_version: &Bound<'_, PyAny>, event_type: &Bound<'_, PyAny>, ) -> PyResult { @@ -50,38 +44,3 @@ fn lifecycle_event_terminal( .map(|entry| entry.terminal) .ok_or_else(|| PyKeyError::new_err(event_type.clone().unbind())) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - let py = module.py(); - module.add( - "CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION", - CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, - )?; - module.add( - "CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION", - CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, - )?; - module.add( - "LIFECYCLE_EVENT_SCHEMA_VERSIONS", - PyFrozenSet::new(py, LIFECYCLE_EVENT_SCHEMA_VERSIONS)?, - )?; - let sets: [(&str, &[&str]); 12] = [ - ("HOOK_PHASES", &HOOK_PHASES), - ("HOOK_TYPES", &HOOK_TYPES), - ("OPERATION_ADAPTERS", &OPERATION_ADAPTERS), - ("OPERATION_KINDS", &OPERATION_KINDS), - ("OPERATION_METADATA_FIELDS", &OPERATION_METADATA_FIELDS), - ("OPERATION_NAMES", &OPERATION_NAMES), - ("OPERATION_PHASES", &OPERATION_PHASES), - ("OPERATION_SCOPES", &OPERATION_SCOPES), - ("OPERATION_STRATEGIES", &OPERATION_STRATEGIES), - ("OPERATION_TARGET_KINDS", &OPERATION_TARGET_KINDS), - ("RESOURCE_SKIP_CODES", &RESOURCE_SKIP_CODES), - ("RESOURCE_SKIP_MODES", &RESOURCE_SKIP_MODES), - ]; - for (name, values) in sets { - module.add(name, PyFrozenSet::new(py, values)?)?; - } - module.add_function(wrap_pyfunction!(lifecycle_event_catalogued, module)?)?; - module.add_function(wrap_pyfunction!(lifecycle_event_terminal, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs index e1e7a6eb6d..a7a4410d8c 100644 --- a/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs @@ -1,11 +1,11 @@ //! Lifecycle event and diagnostic log envelopes, run bounds and operation field rules. use pyo3::exceptions::PyTypeError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{ PyBool, PyByteArray, PyBytes, PyDict, PyDictMethods, PyInt, PyString, PyTypeMethods, }; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::lifecycle_events::constants::INVOCATION_SEQUENCE_PROBLEM; use sqlbuild_core::lifecycle_events::main::diagnostic_severity_problem::diagnostic_severity_problem; use sqlbuild_core::lifecycle_events::main::diagnostic_version_problem::diagnostic_version_problem; @@ -18,12 +18,10 @@ use sqlbuild_core::lifecycle_events::main::run_bounds_problem::run_bounds_proble use sqlbuild_core::lifecycle_events::models::{HookFacts, SequenceFact}; use crate::lifecycle_events::_helpers::payloads::validation_error; -use crate::lifecycle_events::_helpers::validation::frozen_json; -use crate::lifecycle_events::_helpers::validation::known_event_check; -use crate::lifecycle_events::_helpers::validation::optional_text_check; -use crate::lifecycle_events::_helpers::validation::required_text_check; -use crate::lifecycle_events::_helpers::validation::schema_version_check; -use crate::lifecycle_events::_helpers::validation::timestamp_check; +use crate::lifecycle_events::_helpers::validation::{ + frozen_json, known_event_check, optional_text_check, required_text_check, schema_version_check, + timestamp_check, +}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; @@ -106,7 +104,7 @@ fn freeze_object_field(owner: &Bound<'_, PyAny>, field: &str, path: &str) -> PyR /// Validate and freeze one lifecycle event envelope in place. #[pyfunction(name = "validate_lifecycle_event")] #[pyo3(signature = (*, event))] -fn lifecycle_event_check(event: &Bound<'_, PyAny>) -> PyResult<()> { +pub(crate) fn lifecycle_event_check(event: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = event.py(); for field in EVENT_REQUIRED_TEXT { required_text_check(&native_field(event, field)?, field)?; @@ -139,7 +137,7 @@ fn lifecycle_event_check(event: &Bound<'_, PyAny>) -> PyResult<()> { /// Validate a dispatcher-assigned invocation sequence. #[pyfunction(name = "validate_lifecycle_invocation_sequence")] #[pyo3(signature = (*, value))] -fn invocation_sequence_check(value: &Bound<'_, PyAny>) -> PyResult<()> { +pub(crate) fn invocation_sequence_check(value: &Bound<'_, PyAny>) -> PyResult<()> { if !nonnegative_int(value)? { return Err(validation_error(value.py(), INVOCATION_SEQUENCE_PROBLEM)); } @@ -149,7 +147,7 @@ fn invocation_sequence_check(value: &Bound<'_, PyAny>) -> PyResult<()> { /// Validate and freeze one diagnostic log record in place. #[pyfunction(name = "validate_diagnostic_log")] #[pyo3(signature = (*, record))] -fn diagnostic_log_check(record: &Bound<'_, PyAny>) -> PyResult<()> { +pub(crate) fn diagnostic_log_check(record: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = record.py(); let schema_version: Bound<'_, PyAny> = native_field(record, "schema_version")?; schema_version_check(&schema_version)?; @@ -173,7 +171,7 @@ fn diagnostic_log_check(record: &Bound<'_, PyAny>) -> PyResult<()> { /// Validate a run's selection, concurrency and worker counts. #[pyfunction(name = "validate_run_bounds")] #[pyo3(signature = (*, selected_count, configured_concurrency, worker_count))] -fn run_bounds_check( +pub(crate) fn run_bounds_check( selected_count: &Bound<'_, PyAny>, configured_concurrency: &Bound<'_, PyAny>, worker_count: &Bound<'_, PyAny>, @@ -205,7 +203,7 @@ fn run_bounds_check( /// The allow-listed, non-negative operation metadata counts. #[pyfunction] #[pyo3(signature = (*, metadata))] -fn validated_operation_metadata<'py>( +pub(crate) fn validated_operation_metadata<'py>( py: Python<'py>, metadata: &Bound<'py, PyAny>, ) -> PyResult> { @@ -253,7 +251,7 @@ fn hook_name_publishable(name: &Bound<'_, PyAny>) -> PyResult { /// The validated hook fields of an operation, empty when none are supplied. #[pyfunction] #[pyo3(signature = (*, hook_phase, hook_index, hook_type, hook_name))] -fn validated_operation_hook_fields<'py>( +pub(crate) fn validated_operation_hook_fields<'py>( hook_phase: &Bound<'py, PyAny>, hook_index: &Bound<'py, PyAny>, hook_type: &Bound<'py, PyAny>, @@ -289,7 +287,7 @@ fn validated_operation_hook_fields<'py>( /// Validate that a producer other than SQLBuild names its version. #[pyfunction(name = "validate_lifecycle_producer")] #[pyo3(signature = (*, producer, producer_version))] -fn producer_check( +pub(crate) fn producer_check( producer: &Bound<'_, PyAny>, producer_version: &Bound<'_, PyAny>, ) -> PyResult<()> { @@ -298,13 +296,3 @@ fn producer_check( None => Ok(()), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(lifecycle_event_check, module)?)?; - module.add_function(wrap_pyfunction!(invocation_sequence_check, module)?)?; - module.add_function(wrap_pyfunction!(diagnostic_log_check, module)?)?; - module.add_function(wrap_pyfunction!(run_bounds_check, module)?)?; - module.add_function(wrap_pyfunction!(validated_operation_metadata, module)?)?; - module.add_function(wrap_pyfunction!(validated_operation_hook_fields, module)?)?; - module.add_function(wrap_pyfunction!(producer_check, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs index 1c32277205..b2bc109d64 100644 --- a/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs @@ -1,15 +1,14 @@ //! Envelope field validation, JSON freezing and catalogued event validation. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::{PyBool, PyInt, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::lifecycle_events::main::validate_known_event::validate_known_event; use sqlbuild_core::lifecycle_events::models::KnownEventFacts; -use crate::lifecycle_events::_helpers::payloads::PathSegment; -use crate::lifecycle_events::_helpers::payloads::frozen; -use crate::lifecycle_events::_helpers::payloads::object_entries; -use crate::lifecycle_events::_helpers::payloads::validation_error; +use crate::lifecycle_events::_helpers::payloads::{ + PathSegment, frozen, object_entries, validation_error, +}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -102,11 +101,3 @@ pub(crate) fn known_event_check(event: &Bound<'_, PyAny>) -> PyResult<()> { }; validate_known_event(&facts).map_err(|message| validation_error(event.py(), &message)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(schema_version_check, module)?)?; - module.add_function(wrap_pyfunction!(frozen_json, module)?)?; - module.add_function(wrap_pyfunction!(required_text_check, module)?)?; - module.add_function(wrap_pyfunction!(optional_text_check, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_catalog.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_catalog.rs index 9d6001c23a..704fe846de 100644 --- a/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_catalog.rs +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_catalog.rs @@ -1,12 +1,49 @@ //! Register the catalog functions and classes. -use crate::lifecycle_events::_helpers::catalog; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::lifecycle_events::_helpers::catalog::lifecycle_event_catalogued; +use crate::lifecycle_events::_helpers::catalog::lifecycle_event_terminal; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyFrozenSet; +use pyo3::wrap_pyfunction; +use sqlbuild_core::lifecycle_events::constants::{ + CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, HOOK_PHASES, + HOOK_TYPES, LIFECYCLE_EVENT_SCHEMA_VERSIONS, OPERATION_ADAPTERS, OPERATION_KINDS, + OPERATION_METADATA_FIELDS, OPERATION_NAMES, OPERATION_PHASES, OPERATION_SCOPES, + OPERATION_STRATEGIES, OPERATION_TARGET_KINDS, RESOURCE_SKIP_CODES, RESOURCE_SKIP_MODES, +}; /// Register the catalog functions and classes. -#[inline] pub fn register_catalog(module: &Bound<'_, PyModule>) -> PyResult<()> { - catalog::register(module) + let py = module.py(); + module.add( + "CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION", + CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, + )?; + module.add( + "CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION", + CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, + )?; + module.add( + "LIFECYCLE_EVENT_SCHEMA_VERSIONS", + PyFrozenSet::new(py, LIFECYCLE_EVENT_SCHEMA_VERSIONS)?, + )?; + let sets: [(&str, &[&str]); 12] = [ + ("HOOK_PHASES", &HOOK_PHASES), + ("HOOK_TYPES", &HOOK_TYPES), + ("OPERATION_ADAPTERS", &OPERATION_ADAPTERS), + ("OPERATION_KINDS", &OPERATION_KINDS), + ("OPERATION_METADATA_FIELDS", &OPERATION_METADATA_FIELDS), + ("OPERATION_NAMES", &OPERATION_NAMES), + ("OPERATION_PHASES", &OPERATION_PHASES), + ("OPERATION_SCOPES", &OPERATION_SCOPES), + ("OPERATION_STRATEGIES", &OPERATION_STRATEGIES), + ("OPERATION_TARGET_KINDS", &OPERATION_TARGET_KINDS), + ("RESOURCE_SKIP_CODES", &RESOURCE_SKIP_CODES), + ("RESOURCE_SKIP_MODES", &RESOURCE_SKIP_MODES), + ]; + for (name, values) in sets { + module.add(name, PyFrozenSet::new(py, values)?)?; + } + module.add_function(wrap_pyfunction!(lifecycle_event_catalogued, module)?)?; + module.add_function(wrap_pyfunction!(lifecycle_event_terminal, module)?) } diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_envelopes.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_envelopes.rs index 7c70eed1e1..7b902515f6 100644 --- a/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_envelopes.rs +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_envelopes.rs @@ -1,12 +1,22 @@ //! Register the envelopes functions and classes. -use crate::lifecycle_events::_helpers::envelopes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::lifecycle_events::_helpers::envelopes::diagnostic_log_check; +use crate::lifecycle_events::_helpers::envelopes::invocation_sequence_check; +use crate::lifecycle_events::_helpers::envelopes::lifecycle_event_check; +use crate::lifecycle_events::_helpers::envelopes::producer_check; +use crate::lifecycle_events::_helpers::envelopes::run_bounds_check; +use crate::lifecycle_events::_helpers::envelopes::validated_operation_hook_fields; +use crate::lifecycle_events::_helpers::envelopes::validated_operation_metadata; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the envelopes functions and classes. -#[inline] pub fn register_envelopes(module: &Bound<'_, PyModule>) -> PyResult<()> { - envelopes::register(module) + module.add_function(wrap_pyfunction!(lifecycle_event_check, module)?)?; + module.add_function(wrap_pyfunction!(invocation_sequence_check, module)?)?; + module.add_function(wrap_pyfunction!(diagnostic_log_check, module)?)?; + module.add_function(wrap_pyfunction!(run_bounds_check, module)?)?; + module.add_function(wrap_pyfunction!(validated_operation_metadata, module)?)?; + module.add_function(wrap_pyfunction!(validated_operation_hook_fields, module)?)?; + module.add_function(wrap_pyfunction!(producer_check, module)?) } diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs index 7717df24f4..40cd95ae55 100644 --- a/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs @@ -1,12 +1,17 @@ //! Register the validation functions and classes. -use crate::lifecycle_events::_helpers::validation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::lifecycle_events::_helpers::validation::frozen_json; +use crate::lifecycle_events::_helpers::validation::optional_text_check; +use crate::lifecycle_events::_helpers::validation::required_text_check; +use crate::lifecycle_events::_helpers::validation::schema_version_check; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the validation functions and classes. -#[inline] pub fn register_validation(module: &Bound<'_, PyModule>) -> PyResult<()> { - validation::register(module) + module.add_function(wrap_pyfunction!(schema_version_check, module)?)?; + module.add_function(wrap_pyfunction!(frozen_json, module)?)?; + module.add_function(wrap_pyfunction!(required_text_check, module)?)?; + module.add_function(wrap_pyfunction!(optional_text_check, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs b/crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs index a6ed3997df..6ece8f8a74 100644 --- a/crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs +++ b/crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs @@ -32,11 +32,6 @@ pub(crate) fn optional_text(value: &Bound<'_, PyAny>) -> PyResult Ok(Some(value.extract()?)) } -/// `value.timestamp()` of a `datetime`. -pub(crate) fn timestamp(value: &Bound<'_, PyAny>) -> PyResult { - value.call_method0("timestamp")?.extract() -} - /// Raw cursor bounds of one cursor type, parsed once per distinct text. pub(crate) struct BoundParser<'py> { session: CursorSession<'py>, diff --git a/crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs b/crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs index fc1e9012d4..1323a18b41 100644 --- a/crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs +++ b/crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs @@ -1,8 +1,9 @@ //! Coverage, replay-requirement and latest-requirement projections over append-only events. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::microbatches::main::timestamp::timestamp; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_state::microbatches::main::latest_requirement::latest_requirement; use sqlbuild_state::microbatches::main::project_coverage::project_coverage; @@ -12,10 +13,7 @@ use sqlbuild_state::microbatches::models::{ ReplayState, ReplayVersions, RequirementFact, TextInterval, }; -use crate::microbatches::_helpers::conversion::BoundParser; -use crate::microbatches::_helpers::conversion::enum_text; -use crate::microbatches::_helpers::conversion::optional_text; -use crate::microbatches::_helpers::conversion::timestamp; +use crate::microbatches::_helpers::conversion::{BoundParser, enum_text, optional_text}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -108,7 +106,7 @@ fn python_coverage<'py>( /// Fold completion facts and classify uncovered expected intervals. #[pyfunction(name = "project_microbatch_coverage")] #[pyo3(signature = (*, events, expected_intervals, cursor_type))] -fn coverage<'py>( +pub(crate) fn coverage<'py>( events: &Bound<'py, PyAny>, expected_intervals: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, @@ -166,7 +164,7 @@ fn replay_state_member(state: ReplayState) -> &'static str { /// Derive replay completion without writing mutable requirement status. #[pyfunction(name = "project_replay_requirement")] #[pyo3(signature = (**request))] -fn replay<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn replay<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { let request = Request::new("project_replay_requirement", request)?; let py: Python<'py> = request.py(); let requirement: Bound<'py, PyAny> = request.get("requirement")?; @@ -226,7 +224,7 @@ fn replay<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult( +pub(crate) fn latest<'py>( events: &Bound<'py, PyAny>, current_model_version_hash: &str, ) -> PyResult>> { @@ -252,9 +250,3 @@ fn latest<'py>( } Ok(latest_requirement(&facts, current_model_version_hash).map(|index| objects[index].clone())) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(coverage, module)?)?; - module.add_function(wrap_pyfunction!(replay, module)?)?; - module.add_function(wrap_pyfunction!(latest, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs b/crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs index ed75d6dad8..688f4379bb 100644 --- a/crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs +++ b/crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs @@ -1,8 +1,8 @@ //! Direct-mode microbatch state DDL, idempotent inserts, reads and event IDs. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::MICROBATCH_TABLE_NAME; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use sqlbuild_state::events::main::microbatch_event_id::microbatch_event_id; @@ -14,8 +14,7 @@ use sqlbuild_state::microbatches::models::{MicrobatchEventIdentity, ScopeRead}; use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; use sqlbuild_state::state_tables::models::TableColumn; -use crate::microbatches::_helpers::conversion::optional_text; -use crate::microbatches::_helpers::conversion::state_error; +use crate::microbatches::_helpers::conversion::{optional_text, state_error}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::contracts::main::render_state_sql_literal::render_state_sql_literal; @@ -128,7 +127,7 @@ fn events_table<'py>( /// Build one guarded set-based insert for events in the same state table. #[pyfunction(name = "build_microbatch_insert_many_sql")] #[pyo3(signature = (*, events, render_qualified_name, timestamp_type = None))] -fn insert_many( +pub(crate) fn insert_many( events: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, timestamp_type: Option<&str>, @@ -145,7 +144,7 @@ fn insert_many( /// Build one lookup for event IDs already present before a bulk insert. #[pyfunction(name = "build_microbatch_existing_event_ids_sql")] #[pyo3(signature = (*, events, render_qualified_name))] -fn existing_ids( +pub(crate) fn existing_ids( events: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, ) -> PyResult { @@ -161,7 +160,7 @@ fn existing_ids( /// Build the ordered read of one scope's events, every generation for the wildcard. #[pyfunction(name = "build_microbatch_read_scope_sql")] #[pyo3(signature = (*, scope, render_qualified_name))] -fn read_scope( +pub(crate) fn read_scope( scope: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, ) -> PyResult { @@ -191,7 +190,7 @@ fn read_scope( /// Build direct microbatch-state DDL with the adapter's text and timestamp types. #[pyfunction(name = "build_microbatch_create_table_sql")] #[pyo3(signature = (**request))] -fn create_table(request: Option<&Bound<'_, PyDict>>) -> PyResult { +pub(crate) fn create_table(request: Option<&Bound<'_, PyDict>>) -> PyResult { let request = Request::new("build_microbatch_create_table_sql", request)?; let py: Python<'_> = request.py(); let table: String = qualified_table( @@ -228,7 +227,9 @@ fn create_table(request: Option<&Bound<'_, PyDict>>) -> PyResult { /// Build non-unique scope and provenance indexes for indexed warehouses. #[pyfunction(name = "build_microbatch_create_index_sqls")] #[pyo3(signature = (**request))] -fn create_indexes<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn create_indexes<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("build_microbatch_create_index_sqls", request)?; let py: Python<'_> = request.py(); let schema: Bound<'_, PyAny> = request.get("schema")?; @@ -265,7 +266,7 @@ fn create_indexes<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult>) -> PyResult { +pub(crate) fn deterministic_event_id(request: Option<&Bound<'_, PyDict>>) -> PyResult { let request = Request::new("deterministic_microbatch_event_id", request)?; let scope: Bound<'_, PyAny> = request.get("scope")?; let field = |name: &str| -> PyResult { native_field(&scope, name)?.extract() }; @@ -286,12 +287,3 @@ fn deterministic_event_id(request: Option<&Bound<'_, PyDict>>) -> PyResult) -> PyResult<()> { - module.add_function(wrap_pyfunction!(insert_many, module)?)?; - module.add_function(wrap_pyfunction!(existing_ids, module)?)?; - module.add_function(wrap_pyfunction!(read_scope, module)?)?; - module.add_function(wrap_pyfunction!(create_table, module)?)?; - module.add_function(wrap_pyfunction!(create_indexes, module)?)?; - module.add_function(wrap_pyfunction!(deterministic_event_id, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs b/crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs index 4652b5c024..f10c287d0c 100644 --- a/crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs +++ b/crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs @@ -1,12 +1,14 @@ //! Register the projection functions and classes. -use crate::microbatches::_helpers::projection; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::microbatches::_helpers::projection::coverage; +use crate::microbatches::_helpers::projection::latest; +use crate::microbatches::_helpers::projection::replay; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the projection functions and classes. -#[inline] pub fn register_projection(module: &Bound<'_, PyModule>) -> PyResult<()> { - projection::register(module) + module.add_function(wrap_pyfunction!(coverage, module)?)?; + module.add_function(wrap_pyfunction!(replay, module)?)?; + module.add_function(wrap_pyfunction!(latest, module)?) } diff --git a/crates/sqlbuild-python-planner/src/microbatches/main/register_statements.rs b/crates/sqlbuild-python-planner/src/microbatches/main/register_statements.rs index 42b4cab7d5..4e10e3c4a3 100644 --- a/crates/sqlbuild-python-planner/src/microbatches/main/register_statements.rs +++ b/crates/sqlbuild-python-planner/src/microbatches/main/register_statements.rs @@ -1,12 +1,20 @@ //! Register the statements functions and classes. -use crate::microbatches::_helpers::statements; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::microbatches::_helpers::statements::create_indexes; +use crate::microbatches::_helpers::statements::create_table; +use crate::microbatches::_helpers::statements::deterministic_event_id; +use crate::microbatches::_helpers::statements::existing_ids; +use crate::microbatches::_helpers::statements::insert_many; +use crate::microbatches::_helpers::statements::read_scope; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the statements functions and classes. -#[inline] pub fn register_statements(module: &Bound<'_, PyModule>) -> PyResult<()> { - statements::register(module) + module.add_function(wrap_pyfunction!(insert_many, module)?)?; + module.add_function(wrap_pyfunction!(existing_ids, module)?)?; + module.add_function(wrap_pyfunction!(read_scope, module)?)?; + module.add_function(wrap_pyfunction!(create_table, module)?)?; + module.add_function(wrap_pyfunction!(create_indexes, module)?)?; + module.add_function(wrap_pyfunction!(deterministic_event_id, module)?) } diff --git a/crates/sqlbuild-python-planner/src/microbatches/main/timestamp.rs b/crates/sqlbuild-python-planner/src/microbatches/main/timestamp.rs index ceb639b4ac..d73192a825 100644 --- a/crates/sqlbuild-python-planner/src/microbatches/main/timestamp.rs +++ b/crates/sqlbuild-python-planner/src/microbatches/main/timestamp.rs @@ -1,12 +1,8 @@ //! `value.timestamp()` of a `datetime`. -use crate::microbatches::_helpers::conversion; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// `value.timestamp()` of a `datetime`. -#[inline] pub fn timestamp(value: &Bound<'_, PyAny>) -> PyResult { - conversion::timestamp(value) + value.call_method0("timestamp")?.extract() } diff --git a/crates/sqlbuild-python-planner/src/observations/_helpers/publication.rs b/crates/sqlbuild-python-planner/src/observations/_helpers/publication.rs index 69b75275fd..f1971faabe 100644 --- a/crates/sqlbuild-python-planner/src/observations/_helpers/publication.rs +++ b/crates/sqlbuild-python-planner/src/observations/_helpers/publication.rs @@ -9,7 +9,6 @@ use sqlbuild_analysis::planner_warehouse::models::CursorTagRequest; use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND}; use crate::consumption::main::relation::relation; -use crate::observations::_helpers::source_capture::take_source_capture; use crate::planner_warehouse::constants::RUN_BOUNDED; use crate::planner_warehouse::main::inspection::adapter_cursor_sql::adapter_cursor_sql; use crate::planner_warehouse::main::inspection::bound_task::bound_task; @@ -17,8 +16,8 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::native_host::main::append_observations::append_observations; +use sqlbuild_python_front::native_host::models::Location; use sqlbuild_python_front::native_host::models::ObservationAppend; -use sqlbuild_python_front::native_host::models::{ConsumptionConnection, Location}; use sqlbuild_python_project::compiled_views::main::compiled_models::dep_rows::dep_rows; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; @@ -30,7 +29,7 @@ const SOURCE_DEPENDENCY: &str = "source"; const MAX_OBSERVATION_READS: i64 = 4; /// One successful incremental model's cursor read: its name, state location, cursor type and SQL. -struct CursorRead { +pub(crate) struct CursorRead { model: String, location: Location, cursor_type: String, @@ -38,7 +37,7 @@ struct CursorRead { } /// The cursor read of every successful incremental model, rendered as the planner renders it. -fn cursor_reads( +pub(crate) fn cursor_reads( project: &Bound<'_, PyAny>, adapter: &Bound<'_, PyAny>, successful_models: &[String], @@ -83,7 +82,7 @@ fn cursor_reads( } /// One `source_columns` row per described source, in the state schema of each model reading it. -fn source_column_rows( +pub(crate) fn source_column_rows( project: &Bound<'_, PyAny>, described: &BTreeMap, run_id: &str, @@ -127,7 +126,7 @@ fn source_column_rows( } /// Read each cursor `MAX`, then append it with `rows`, by state location; one warning per failure. -fn observe<'py>( +pub(crate) fn observe<'py>( reads: &[CursorRead], mut rows: BTreeMap>, (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -186,34 +185,3 @@ fn observe<'py>( } Ok(warnings) } - -/// Record successful incremental models' cursor `MAX` and the source columns planning described; failures only warn. -pub(crate) fn append_build_observations( - (project, successful_models): (&Bound<'_, PyAny>, &[String]), - (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - run_id: &str, -) -> Vec { - let described: BTreeMap = take_source_capture().into_iter().collect(); - let recorded: PyResult> = (|| { - let reads: Vec = cursor_reads(project, adapter, successful_models)?; - let sources: BTreeMap> = - source_column_rows(project, &described, run_id)?; - if reads.is_empty() && sources.is_empty() { - return Ok(Vec::new()); - } - let py: Python<'_> = adapter.py(); - let connection = ConsumptionConnection::new(adapter, connection_config); - let observed: PyResult> = connection - .connected(py) - .and_then(|handle| observe(&reads, sources, (adapter, &handle), run_id)); - let closed: PyResult<()> = connection.close(py); - let warnings: Vec = observed?; - closed?; - Ok(warnings) - })(); - recorded.unwrap_or_else(|error| { - vec![format!( - "could not record observations: {error}; plan previews show them as unknown" - )] - }) -} diff --git a/crates/sqlbuild-python-planner/src/observations/_helpers/source_capture.rs b/crates/sqlbuild-python-planner/src/observations/_helpers/source_capture.rs index b33192b5fd..78ca319182 100644 --- a/crates/sqlbuild-python-planner/src/observations/_helpers/source_capture.rs +++ b/crates/sqlbuild-python-planner/src/observations/_helpers/source_capture.rs @@ -2,46 +2,10 @@ use std::cell::RefCell; -use serde_json::{Map, Value}; -use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; -use sqlbuild_state::observations::constants::{ - SOURCE_COLUMNS_NAMES_KEY, SOURCE_COLUMNS_SQL_HASH_KEY, -}; - /// Each described source's name and observation value, in the order planning described them. pub(crate) type DescribedSources = Vec<(String, String)>; thread_local! { /// Per thread, since a build plans and records on the thread that runs it; `None` when not armed. - static CAPTURED: RefCell> = const { RefCell::new(None) }; -} - -/// Start capturing described source columns; only a build arms it, so `sqb plan` records nothing. -pub(crate) fn arm_source_capture() { - CAPTURED.with(|slot| *slot.borrow_mut() = Some(Vec::new())); -} - -/// Keep one source's described SQL hash and column names when a build armed the capture. -pub(crate) fn capture_source_columns(source: &str, sql: &str, columns: &[String]) { - CAPTURED.with(|slot| { - if let Some(captured) = slot.borrow_mut().as_mut() { - let mut value: Map = Map::new(); - value.insert( - SOURCE_COLUMNS_SQL_HASH_KEY.to_owned(), - Value::String(sha256_hex(sql.as_bytes())), - ); - value.insert( - SOURCE_COLUMNS_NAMES_KEY.to_owned(), - Value::from(columns.to_vec()), - ); - captured.push((source.to_owned(), Value::Object(value).to_string())); - } - }); -} - -/// The captured source columns, disarming the capture. -pub(crate) fn take_source_capture() -> DescribedSources { - CAPTURED - .with(|slot| slot.borrow_mut().take()) - .unwrap_or_default() + pub(crate) static CAPTURED: RefCell> = const { RefCell::new(None) }; } diff --git a/crates/sqlbuild-python-planner/src/observations/main/append_build_observations.rs b/crates/sqlbuild-python-planner/src/observations/main/append_build_observations.rs index 428c32de9c..d0e653af69 100644 --- a/crates/sqlbuild-python-planner/src/observations/main/append_build_observations.rs +++ b/crates/sqlbuild-python-planner/src/observations/main/append_build_observations.rs @@ -1,19 +1,42 @@ //! Record successful incremental models' cursor `MAX` and the source columns planning described; failures only warn. -use crate::observations::_helpers::publication; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; +use crate::observations::_helpers::publication::CursorRead; +use crate::observations::_helpers::publication::cursor_reads; +use crate::observations::_helpers::publication::observe; +use crate::observations::_helpers::publication::source_column_rows; +use crate::observations::main::take_source_capture::take_source_capture; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_front::native_host::models::ObservationAppend; +use sqlbuild_python_front::native_host::models::{ConsumptionConnection, Location}; +use std::collections::BTreeMap; /// Record successful incremental models' cursor `MAX` and the source columns planning described; failures only warn. -#[inline] pub fn append_build_observations( (project, successful_models): (&Bound<'_, PyAny>, &[String]), (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), run_id: &str, ) -> Vec { - publication::append_build_observations( - (project, successful_models), - (adapter, connection_config), - run_id, - ) + let described: BTreeMap = take_source_capture().into_iter().collect(); + let recorded: PyResult> = (|| { + let reads: Vec = cursor_reads(project, adapter, successful_models)?; + let sources: BTreeMap> = + source_column_rows(project, &described, run_id)?; + if reads.is_empty() && sources.is_empty() { + return Ok(Vec::new()); + } + let py: Python<'_> = adapter.py(); + let connection = ConsumptionConnection::new(adapter, connection_config); + let observed: PyResult> = connection + .connected(py) + .and_then(|handle| observe(&reads, sources, (adapter, &handle), run_id)); + let closed: PyResult<()> = connection.close(py); + let warnings: Vec = observed?; + closed?; + Ok(warnings) + })(); + recorded.unwrap_or_else(|error| { + vec![format!( + "could not record observations: {error}; plan previews show them as unknown" + )] + }) } diff --git a/crates/sqlbuild-python-planner/src/observations/main/arm_source_capture.rs b/crates/sqlbuild-python-planner/src/observations/main/arm_source_capture.rs index 1f1448862a..df8a0a1959 100644 --- a/crates/sqlbuild-python-planner/src/observations/main/arm_source_capture.rs +++ b/crates/sqlbuild-python-planner/src/observations/main/arm_source_capture.rs @@ -1,9 +1,8 @@ //! Start capturing described source columns; only a build arms it, so `sqb plan` records nothing. -use crate::observations::_helpers::source_capture; +use crate::observations::_helpers::source_capture::CAPTURED; /// Start capturing described source columns; only a build arms it, so `sqb plan` records nothing. -#[inline] pub fn arm_source_capture() { - source_capture::arm_source_capture() + CAPTURED.with(|slot| *slot.borrow_mut() = Some(Vec::new())); } diff --git a/crates/sqlbuild-python-planner/src/observations/main/capture_source_columns.rs b/crates/sqlbuild-python-planner/src/observations/main/capture_source_columns.rs index af1b51edac..e854c03d54 100644 --- a/crates/sqlbuild-python-planner/src/observations/main/capture_source_columns.rs +++ b/crates/sqlbuild-python-planner/src/observations/main/capture_source_columns.rs @@ -1,9 +1,26 @@ //! Keep one source's described SQL hash and column names when a build armed the capture. -use crate::observations::_helpers::source_capture; +use crate::observations::_helpers::source_capture::CAPTURED; +use serde_json::{Map, Value}; +use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; +use sqlbuild_state::observations::constants::{ + SOURCE_COLUMNS_NAMES_KEY, SOURCE_COLUMNS_SQL_HASH_KEY, +}; /// Keep one source's described SQL hash and column names when a build armed the capture. -#[inline] pub fn capture_source_columns(source: &str, sql: &str, columns: &[String]) { - source_capture::capture_source_columns(source, sql, columns) + CAPTURED.with(|slot| { + if let Some(captured) = slot.borrow_mut().as_mut() { + let mut value: Map = Map::new(); + value.insert( + SOURCE_COLUMNS_SQL_HASH_KEY.to_owned(), + Value::String(sha256_hex(sql.as_bytes())), + ); + value.insert( + SOURCE_COLUMNS_NAMES_KEY.to_owned(), + Value::from(columns.to_vec()), + ); + captured.push((source.to_owned(), Value::Object(value).to_string())); + } + }); } diff --git a/crates/sqlbuild-python-planner/src/observations/main/take_source_capture.rs b/crates/sqlbuild-python-planner/src/observations/main/take_source_capture.rs index e9d6c47289..5e0175e45d 100644 --- a/crates/sqlbuild-python-planner/src/observations/main/take_source_capture.rs +++ b/crates/sqlbuild-python-planner/src/observations/main/take_source_capture.rs @@ -1,10 +1,11 @@ //! The captured source columns, disarming the capture. -use crate::observations::_helpers::source_capture; +use crate::observations::_helpers::source_capture::CAPTURED; use crate::observations::_helpers::source_capture::DescribedSources; /// The captured source columns, disarming the capture. -#[inline] pub fn take_source_capture() -> DescribedSources { - source_capture::take_source_capture() + CAPTURED + .with(|slot| slot.borrow_mut().take()) + .unwrap_or_default() } diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs index c194f877f0..6f1ef98fbb 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs @@ -1,12 +1,12 @@ //! One planned model's requested, resolved and deferred cursor state, for text and JSON plans. +use crate::plan_output::main::aligned_timestamp_bounds::aligned_timestamp_bounds; +use crate::plan_output::main::resolve_effective_microbatch_batch_size::resolve_effective_microbatch_batch_size; +use crate::plan_output::main::text::text; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::plan_output::_helpers::display::aligned_timestamp_bounds; -use crate::plan_output::_helpers::display::resolve_effective_microbatch_batch_size; use crate::plan_output::_helpers::values::optional_text; -use crate::plan_output::_helpers::values::text; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs index 03633e50b2..962c5d026d 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs @@ -1,37 +1,28 @@ //! Plan and progress display fragments over planner entries and cursor values. +use crate::plan_output::main::text::text; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyString}; use sqlbuild_core::plan_display::constants::{ DEFAULT_CURSOR_GRAIN, INTEGER_CURSOR, TIMESTAMP_CURSOR, }; -use sqlbuild_core::plan_display::main::cursor_bound_display::cursor_bound_display as bound_display; -use sqlbuild_core::plan_display::main::effective_microbatch_batch_size::effective_microbatch_batch_size; use sqlbuild_core::plan_display::main::format_no_input_rows::format_no_input_rows as no_input_rows; use sqlbuild_core::plan_display::main::format_waiting_on_empty_inputs::format_waiting_on_empty_inputs as waiting_on_empty_inputs; use sqlbuild_core::plan_display::main::model_materialization_label::model_materialization_label as materialization_label; use sqlbuild_core::plan_display::models::{ModelLabelFacts, ParsedBound}; use crate::plan_output::_helpers::values::optional_text; -use crate::plan_output::_helpers::values::text; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; -use sqlbuild_python_front::cursor::main::scalars::to_exclusive::to_exclusive; -use sqlbuild_python_front::cursor::main::scalars::to_inclusive::to_inclusive; -use sqlbuild_python_front::cursor::main::scalars::try_parse::try_parse; const DATETIME_MODULE: &str = "datetime"; -fn label_facts(entry: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn label_facts(entry: &Bound<'_, PyAny>) -> PyResult { let field = |name: &str| -> PyResult> { optional_text(&native_field(entry, name)?) }; Ok(ModelLabelFacts { @@ -47,15 +38,8 @@ fn label_facts(entry: &Bound<'_, PyAny>) -> PyResult { }) } -pub(crate) fn optional_facts(entry: &Bound<'_, PyAny>) -> PyResult> { - if entry.is_none() { - return Ok(None); - } - label_facts(entry).map(Some) -} - /// The cursor type, defaulting to timestamp, and its grain object (`None` for integers). -fn cursor_typing<'py>( +pub(crate) fn cursor_typing<'py>( cursor_type: &Bound<'py, PyAny>, cursor_grain: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { @@ -77,48 +61,7 @@ fn cursor_typing<'py>( Ok((effective, grain)) } -/// The last cursor value included by an exclusive end bound. -#[pyfunction] -#[pyo3(signature = (*, end, cursor_type, cursor_grain))] -pub(crate) fn inclusive_cursor_end( - end: &Bound<'_, PyAny>, - cursor_type: &Bound<'_, PyAny>, - cursor_grain: &Bound<'_, PyAny>, -) -> PyResult { - let (effective, grain) = cursor_typing(cursor_type, cursor_grain)?; - let Some(parsed) = try_parse(end, &effective)? else { - return end.extract(); - }; - render_value(&to_inclusive(&parsed, &grain)?) -} - -/// A fully bounded inclusive operator override as an executable interval. -pub(crate) fn resolve_bounded_cursor_override<'py>( - start_cursor_override: &Bound<'py, PyAny>, - end_cursor_override: &Bound<'py, PyAny>, - cursor_type: &Bound<'py, PyAny>, - cursor_grain: &Bound<'py, PyAny>, -) -> PyResult>> { - let py: Python<'py> = start_cursor_override.py(); - if start_cursor_override.is_none() || end_cursor_override.is_none() { - return Ok(None); - } - let (effective, grain) = cursor_typing(cursor_type, cursor_grain)?; - let (Some(start), Some(end)) = ( - try_parse(start_cursor_override, &effective)?, - try_parse(end_cursor_override, &effective)?, - ) else { - return Ok(None); - }; - new_record_with( - py, - RecordId::CursorBounds, - &[("start", start), ("end", to_exclusive(&end, &grain)?)], - ) - .map(Some) -} - -fn parsed_bound(value: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn parsed_bound(value: &Bound<'_, PyAny>) -> PyResult> { let py: Python<'_> = value.py(); let parsed: Bound<'_, PyAny> = match imported(py, DATETIME_MODULE, "datetime")?.call_method1("fromisoformat", (value,)) { @@ -139,31 +82,6 @@ fn parsed_bound(value: &Bound<'_, PyAny>) -> PyResult> { })) } -/// A cursor bound for display, collapsing a whole-day midnight timestamp to a date. -#[pyfunction] -#[pyo3(signature = (*, value, cursor_type, cursor_grain))] -pub(crate) fn cursor_bound_display( - value: &Bound<'_, PyAny>, - cursor_type: &Bound<'_, PyAny>, - cursor_grain: &Bound<'_, PyAny>, -) -> PyResult { - let cursor_type: Option = optional_text(cursor_type)?; - let raw: String = value.extract()?; - let parsed: Option = if cursor_type.as_deref() == Some(INTEGER_CURSOR) { - None - } else { - parsed_bound(value)? - }; - Ok(bound_display( - &raw, - ( - cursor_type.as_deref(), - optional_text(cursor_grain)?.as_deref(), - ), - parsed.as_ref(), - )) -} - /// The full materialization label used in plan summaries. pub(crate) fn model_materialization_label(entry: &Bound<'_, PyAny>) -> PyResult { Ok(materialization_label(&label_facts(entry)?)) @@ -187,23 +105,7 @@ pub(crate) fn format_waiting_on_empty_inputs(input_names: &Bound<'_, PyAny>) -> Ok(waiting_on_empty_inputs(&names(input_names)?)) } -/// The explicit effective batch-size token resolved against the replay grain. -#[pyfunction] -#[pyo3(signature = (*, batch_size, effective_grain))] -pub(crate) fn resolve_effective_microbatch_batch_size( - py: Python<'_>, - batch_size: &str, - effective_grain: &str, -) -> Result { - effective_microbatch_batch_size(batch_size, effective_grain).map_err(|error| { - record_class(py, RecordId::CursorGrain) - .and_then(|grain| grain.call1((error.0,))) - .err() - .unwrap_or_else(|| PyValueError::new_err("unknown cursor grain")) - }) -} - -fn as_datetime<'py>(value: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn as_datetime<'py>(value: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = value.py(); let inner: Bound<'py, PyAny> = native_field(value, "value")?; if value.is_instance(&record_class(py, RecordId::TimestampValue)?)? { @@ -214,7 +116,7 @@ fn as_datetime<'py>(value: &Bound<'py, PyAny>) -> PyResult> { datetime.call_method1("combine", (inner, midnight)) } -fn presume_utc<'py>(value: Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn presume_utc<'py>(value: Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = value.py(); if !native_field(&value, "tzinfo")?.is_none() { return Ok(value); @@ -227,32 +129,3 @@ fn presume_utc<'py>(value: Bound<'py, PyAny>) -> PyResult> { )?; value.call_method("replace", (), Some(&options)) } - -/// Comparable bounds, presuming a naive side UTC when the other is aware. -#[pyfunction] -#[pyo3(signature = (*, start, end))] -pub(crate) fn aligned_timestamp_bounds<'py>( - start: &Bound<'py, PyAny>, - end: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = start.py(); - let mut start_at: Bound<'py, PyAny> = as_datetime(start)?; - let mut end_at: Bound<'py, PyAny> = as_datetime(end)?; - if native_field(&start_at, "tzinfo")?.is_none() != native_field(&end_at, "tzinfo")?.is_none() { - start_at = presume_utc(start_at)?; - end_at = presume_utc(end_at)?; - } - PyTuple::new(py, [start_at, end_at]) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(inclusive_cursor_end, module)?)?; - module.add_function(wrap_pyfunction!(cursor_bound_display, module)?)?; - module.add_function(wrap_pyfunction!(format_waiting_on_empty_inputs, module)?)?; - module.add_function(wrap_pyfunction!( - resolve_effective_microbatch_batch_size, - module - )?)?; - module.add_function(wrap_pyfunction!(aligned_timestamp_bounds, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs index 1d35c1f3d9..b8bc601539 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs @@ -1,14 +1,11 @@ //! Convert public migration models at the Python boundary for native presentation. use crate::plan_output::_helpers::plan_text::display_limit; -use crate::plan_output::_helpers::values::iso; -use crate::plan_output::_helpers::values::items; -use crate::plan_output::_helpers::values::optional_text; -use crate::plan_output::_helpers::values::relation_name; -use crate::plan_output::_helpers::values::text; -use crate::plan_output::_helpers::values::texts; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::plan_output::_helpers::values::{iso, optional_text, relation_name, texts}; +use crate::plan_output::main::items::items; +use crate::plan_output::main::text::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use sqlbuild_core::cli_text::models::{ CliStyle, ColumnMigrationText, MigrationDecisionText, MigrationText, MigrationTextReport, OldNameText, PreBuildWorkText, RetentionText, TableTypeText, @@ -188,7 +185,7 @@ pub(crate) fn pre_build_lines( /// Format retention, table-type, and migration work that runs before model builds. #[pyfunction(name = "format_pre_build_work", signature = (*, lines, plan, display_options))] -fn format_pre_build( +pub(crate) fn format_pre_build( lines: Vec, plan: &Bound<'_, PyAny>, display_options: &Bound<'_, PyAny>, @@ -197,7 +194,10 @@ fn format_pre_build( } #[pyfunction(name = "old_name_rows", signature = (*, entry, style))] -fn render_old_name(entry: &Bound<'_, PyAny>, style: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn render_old_name( + entry: &Bound<'_, PyAny>, + style: &Bound<'_, PyAny>, +) -> PyResult> { Ok(old_name_rows::old_name_rows( &old_name(entry)?, CliStyle { @@ -205,8 +205,3 @@ fn render_old_name(entry: &Bound<'_, PyAny>, style: &Bound<'_, PyAny>) -> PyResu }, )) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(format_pre_build, module)?)?; - module.add_function(wrap_pyfunction!(render_old_name, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs index 2dafc928e7..58f9c36c29 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs @@ -1,32 +1,22 @@ //! The JSON plan document, read from the planner's output by attribute. -use crate::plan_output::constants::CHANGED_KEY; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::plan_output::main::any::any; +use crate::plan_output::main::dict::dict; +use crate::plan_output::main::future_cursor_safety::future_cursor_safety; +use crate::plan_output::main::items::items; +use crate::plan_output::main::maximum_start_safety::maximum_start_safety; +use crate::plan_output::main::text::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::plan_display::constants::SYNCED_COPY_KEY; -use crate::plan_output::_helpers::cursor_details::CursorDetails; -use crate::plan_output::_helpers::cursor_details::cursor_details; -use crate::plan_output::_helpers::cursor_details::owner; -use crate::plan_output::_helpers::plan_text::model_config_diff; -use crate::plan_output::_helpers::plan_text::model_query_diff; -use crate::plan_output::_helpers::python_identity::definition_source_text; -use crate::plan_output::_helpers::python_identity::dependency_source_text; -use crate::plan_output::_helpers::safety::future_cursor_safety; -use crate::plan_output::_helpers::safety::maximum_start_safety; -use crate::plan_output::_helpers::values::any; -use crate::plan_output::_helpers::values::dict; -use crate::plan_output::_helpers::values::enum_value; -use crate::plan_output::_helpers::values::iso; -use crate::plan_output::_helpers::values::items; -use crate::plan_output::_helpers::values::list; -use crate::plan_output::_helpers::values::mapped; -use crate::plan_output::_helpers::values::optional_enum_value; -use crate::plan_output::_helpers::values::relation_name; -use crate::plan_output::_helpers::values::text; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::compile_outputs::main::indented_json::indented_json; +use crate::plan_output::_helpers::cursor_details::{CursorDetails, cursor_details, owner}; +use crate::plan_output::_helpers::plan_text::{model_config_diff, model_query_diff}; +use crate::plan_output::_helpers::python_identity::{ + definition_source_text, dependency_source_text, +}; +use crate::plan_output::_helpers::values::{ + enum_value, iso, list, mapped, optional_enum_value, relation_name, +}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; @@ -155,7 +145,7 @@ fn microbatch_state<'py>(entry: &Bound<'py, PyAny>) -> PyResult(entry: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn model_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = entry.py(); let model: Bound<'py, PyDict> = dict( py, @@ -303,7 +293,7 @@ fn with_qualified_name<'py>( Ok(item.into_any()) } -fn seed_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn seed_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = entry.py(); with_qualified_name( dict( @@ -317,7 +307,7 @@ fn seed_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { ) } -fn source_load_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn source_load_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = entry.py(); let load: Bound<'py, PyDict> = dict( py, @@ -344,7 +334,7 @@ fn source_load_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult(entry: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn function_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = entry.py(); let columns: Bound<'py, PyAny> = native_field(entry, "return_columns")?; with_qualified_name( @@ -388,7 +378,7 @@ fn function_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> ) } -fn warning_json<'py>(warning: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn warning_json<'py>(warning: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = warning.py(); let item: Bound<'py, PyDict> = dict( py, @@ -461,7 +451,7 @@ fn identity_diff<'py>(entry: &Bound<'py, PyAny>) -> PyResult> Ok(result) } -fn python_entry_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn python_entry_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = entry.py(); let item: Bound<'py, PyDict> = dict( py, @@ -487,7 +477,7 @@ struct Usage<'py> { module: Bound<'py, PyAny>, } -fn providers_json<'py>( +pub(crate) fn providers_json<'py>( plan: &Bound<'py, PyAny>, python_entries: &[Bound<'py, PyAny>], ) -> PyResult> { @@ -565,7 +555,7 @@ fn providers_json<'py>( Ok(providers.into_any()) } -fn old_name_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn old_name_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = entry.py(); Ok(dict( py, @@ -591,7 +581,7 @@ fn old_name_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> .into_any()) } -fn migrations_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn migrations_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = plan.py(); let old_names: Bound<'py, PyDict> = PyDict::new(py); for entry in items(&native_field(plan, "old_name_view_entries")?)? { @@ -667,7 +657,7 @@ fn migrations_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> }) } -fn column_migrations_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn column_migrations_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = plan.py(); mapped(&native_field(plan, "column_migration_entries")?, |entry| { Ok(dict( @@ -694,7 +684,7 @@ fn column_migrations_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult(plan: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn retention_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = plan.py(); mapped(&native_field(plan, "retention_entries")?, |entry| { let request: Bound<'py, PyAny> = native_field(entry, "request")?; @@ -724,7 +714,7 @@ fn retention_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> }) } -fn table_type_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn table_type_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = plan.py(); mapped(&native_field(plan, "table_type_entries")?, |entry| { Ok(dict( @@ -746,103 +736,3 @@ fn table_type_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> .into_any()) }) } - -/// The plan as one JSON document, `json.dumps(indent=2)`. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn format_plan_json( - py: Python<'_>, - options: Option>, -) -> PyResult { - let arguments: KeywordArguments<'_> = KeywordArguments::read( - py, - "format_plan_json", - options, - ( - &["plan"], - &["python_plan_entries", SYNCED_COPY_KEY, CHANGED_KEY], - ), - )?; - let plan: Bound<'_, PyAny> = arguments.value("plan")?; - let python_entries: Vec> = if arguments.passed("python_plan_entries")? { - items(&arguments.value("python_plan_entries")?)? - } else { - Vec::new() - }; - let length = |field: &str| -> PyResult { native_field(&plan, field)?.len() }; - let python_nodes: Vec> = python_entries - .iter() - .map(python_entry_json) - .collect::>()?; - let result: Bound<'_, PyDict> = dict( - py, - vec![ - ( - "selected_count", - any( - py, - length("model_entries")? - + length("seed_entries")? - + length("function_entries")?, - )?, - ), - ( - "source_load_count", - any(py, length("source_load_entries")?)?, - ), - ("python_node_count", any(py, python_nodes.len())?), - ( - "models", - mapped(&native_field(&plan, "model_entries")?, model_json)?, - ), - ( - "seeds", - mapped(&native_field(&plan, "seed_entries")?, seed_json)?, - ), - ( - "source_loads", - mapped( - &native_field(&plan, "source_load_entries")?, - source_load_json, - )?, - ), - ( - "functions", - mapped(&native_field(&plan, "function_entries")?, function_json)?, - ), - ("python_nodes", PyList::new(py, python_nodes)?.into_any()), - ("providers", providers_json(&plan, &python_entries)?), - ( - "warnings", - mapped(&native_field(&plan, "warnings")?, warning_json)?, - ), - ("retention", retention_json(&plan)?), - ("table_type_conversions", table_type_json(&plan)?), - ("migrations", migrations_json(&plan)?), - ("column_migrations", column_migrations_json(&plan)?), - ( - "old_names", - mapped( - &native_field(&plan, "old_name_view_entries")?, - old_name_json, - )?, - ), - ], - )?; - let metadata: Bound<'_, PyAny> = native_field(&plan, "metadata")?; - if metadata.is_truthy()? { - result.set_item("metadata", metadata)?; - } - let synced_copy: Bound<'_, PyAny> = arguments.value(SYNCED_COPY_KEY)?; - if !synced_copy.is_none() { - result.set_item(SYNCED_COPY_KEY, synced_copy)?; - } - if arguments.passed(CHANGED_KEY)? { - result.set_item(CHANGED_KEY, arguments.value(CHANGED_KEY)?)?; - } - indented_json(result.as_any()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(format_plan_json, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index 546f456cdc..9774b889a9 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -1,10 +1,11 @@ //! The text plan: planner objects read by attribute into the native plan report. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::plan_output::main::cursor_bound_display::cursor_bound_display; +use crate::plan_output::main::inclusive_cursor_end::inclusive_cursor_end; +use crate::plan_output::main::items::items; +use crate::plan_output::main::text::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::cli_text::main::format_plan_completion::format_plan_completion; -use sqlbuild_core::cli_text::main::format_plan_text::format_plan_text; use sqlbuild_core::cli_text::models::{ CliStyle, CursorPlanText, DisplayLimit, FunctionPlanText, ModelPlanText, PlanTextReport, PlanWarningText, ProviderUsageText, PythonPlanText, RenameHintText, SchemaFindingText, @@ -12,24 +13,18 @@ use sqlbuild_core::cli_text::models::{ }; use sqlbuild_core::plan_display::main::format_query_diff::format_query_diff; -use crate::plan_output::_helpers::cursor_details::CursorDetails; -use crate::plan_output::_helpers::cursor_details::DEFERRED; -use crate::plan_output::_helpers::cursor_details::RESOLVED; -use crate::plan_output::_helpers::cursor_details::cursor_details; -use crate::plan_output::_helpers::cursor_details::runtime_inputs_clause; -use crate::plan_output::_helpers::display::cursor_bound_display; -use crate::plan_output::_helpers::display::format_no_input_rows; -use crate::plan_output::_helpers::display::format_waiting_on_empty_inputs; -use crate::plan_output::_helpers::display::inclusive_cursor_end; -use crate::plan_output::_helpers::display::model_materialization_label; +use crate::plan_output::_helpers::cursor_details::{ + CursorDetails, DEFERRED, RESOLVED, cursor_details, runtime_inputs_clause, +}; +use crate::plan_output::_helpers::display::{ + format_no_input_rows, format_waiting_on_empty_inputs, model_materialization_label, +}; use crate::plan_output::_helpers::migrations::pre_build_lines; use crate::plan_output::_helpers::plan_json::unified_diff; -use crate::plan_output::_helpers::python_identity::definition_source_text; -use crate::plan_output::_helpers::python_identity::dependency_source_text; -use crate::plan_output::_helpers::values::items; -use crate::plan_output::_helpers::values::optional_text; -use crate::plan_output::_helpers::values::text; -use crate::plan_output::_helpers::values::texts; +use crate::plan_output::_helpers::python_identity::{ + definition_source_text, dependency_source_text, +}; +use crate::plan_output::_helpers::values::{optional_text, texts}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -42,7 +37,7 @@ const QUERY_CHANGED: &str = "query_changed"; const CONFIG_CHANGED: &str = "config_changed"; const MICROBATCH_MODE: &str = "microbatch"; const CHANGED_IDENTITY: &str = "changed"; -const PLAN_OPTIONS: (&[&str], &[&str]) = ( +pub(crate) const PLAN_OPTIONS: (&[&str], &[&str]) = ( &["plan"], &[ "full_refresh", @@ -444,7 +439,7 @@ pub(crate) fn display_limit(options: &Bound<'_, PyAny>) -> PyResult) -> PyResult { +pub(crate) fn report(arguments: &KeywordArguments<'_>) -> PyResult { let py: Python<'_> = arguments.value("plan")?.py(); let plan: Bound<'_, PyAny> = arguments.value("plan")?; let display_options: Bound<'_, PyAny> = { @@ -522,25 +517,3 @@ fn report(arguments: &KeywordArguments<'_>) -> PyResult { pre_build_lines: pre_build, }) } - -/// The text plan grouped by reason with inline detail, optionally closed by its completion line. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn format_plan(py: Python<'_>, options: Option>) -> PyResult { - let arguments: KeywordArguments<'_> = - KeywordArguments::read(py, "format_plan", options, PLAN_OPTIONS)?; - let report: PlanTextReport = report(&arguments)?; - let mut text: String = format_plan_text(&report); - if arguments.passed("changed_sections")? { - let sections: String = arguments.value("changed_sections")?.extract()?; - text = format!("{text}\n\n{sections}"); - } - if !arguments.flag("include_completion")? { - return Ok(text); - } - Ok(format!("{text}\n\n{}", format_plan_completion(&report))) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(format_plan, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/safety.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/safety.rs index 5936609206..8f9e3e8d4f 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/safety.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/safety.rs @@ -1,29 +1,28 @@ //! Structured future-cursor and maximum-start safety evidence for plan and result JSON. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::plan_output::main::dict::dict; +use crate::plan_output::main::maximum_start_safety::maximum_start_safety; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyString; -use crate::plan_output::_helpers::values::dict; -use crate::plan_output::_helpers::values::enum_value; -use crate::plan_output::_helpers::values::items; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; /// `render(value=value)`. -fn render<'py>(value: Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn render<'py>(value: Bound<'py, PyAny>) -> PyResult> { Ok(PyString::new(value.py(), &render_value(&value)?).into_any()) } /// `render(value=value)` when set, else `None`. -fn optional_render<'py>(value: Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn optional_render<'py>(value: Bound<'py, PyAny>) -> PyResult> { if value.is_none() { return Ok(value); } render(value) } -fn bounds<'py>( +pub(crate) fn bounds<'py>( evidence: &Bound<'py, PyAny>, (start, end): (&str, &str), ) -> PyResult> { @@ -37,133 +36,10 @@ fn bounds<'py>( .into_any()) } -/// Cap policy, bounds and physical input evidence of future-cursor safety, or `None`. -pub(crate) fn future_cursor_safety<'py>( - evidence: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = evidence.py(); - if evidence.is_none() { - return Ok(evidence.clone()); - } - let relation: Bound<'py, PyAny> = native_field(evidence, "determining_relation")?; - let determining: Bound<'py, PyAny> = if relation.is_none() { - relation - } else { - dict( - py, - vec![ - ("relation", relation), - ( - "cursor_column", - native_field(evidence, "determining_cursor_column")?, - ), - ], - )? - .into_any() - }; - let mut inputs: Vec> = Vec::new(); - for item in items(&native_field(evidence, "inputs")?)? { - inputs.push(dict( - py, - vec![ - ("relation", native_field(&item, "relation")?), - ("cursor_column", native_field(&item, "cursor_column")?), - ("minimum", optional_render(native_field(&item, "minimum")?)?), - ("maximum", render(native_field(&item, "maximum")?)?), - ], - )?); - } - Ok(dict( - py, - vec![ - ("action", enum_value(evidence, "action")?), - ("max_distance", native_field(evidence, "max_distance")?), - ( - "invocation_time", - render(native_field(evidence, "invocation_time")?)?, - ), - ( - "discovered_bounds", - bounds(evidence, ("discovered_start", "discovered_end"))?, - ), - ( - "applied_bounds", - bounds(evidence, ("applied_start", "applied_end"))?, - ), - ( - "maximum_allowed_bounds", - bounds(evidence, ("maximum_allowed_start", "maximum_allowed_end"))?, - ), - ( - "future_start_detected", - native_field(evidence, "future_start_detected")?, - ), - ( - "future_end_detected", - native_field(evidence, "future_end_detected")?, - ), - ("determining_input", determining), - ("inputs", pyo3::types::PyList::new(py, inputs)?.into_any()), - ], - )? - .into_any()) -} - -/// Automatic-start policy and physical eligibility evidence, or `None`. -pub(crate) fn maximum_start_safety<'py>( - evidence: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = evidence.py(); - if evidence.is_none() { - return Ok(evidence.clone()); - } - Ok(dict( - py, - vec![ - ("action", enum_value(evidence, "action")?), - ("max_ahead", native_field(evidence, "max_ahead")?), - ( - "invocation_time", - render(native_field(evidence, "invocation_time")?)?, - ), - ( - "physical_target_max", - render(native_field(evidence, "physical_target_max")?)?, - ), - ( - "highest_eligible_target_max", - optional_render(native_field(evidence, "highest_eligible_target_max")?)?, - ), - ( - "effective_start", - render(native_field(evidence, "effective_start")?)?, - ), - ( - "maximum_allowed_start", - render(native_field(evidence, "maximum_allowed_start")?)?, - ), - ( - "input", - dict( - py, - vec![ - ("relation", native_field(evidence, "target_relation")?), - ("cursor_column", native_field(evidence, "cursor_column")?), - ], - )? - .into_any(), - ), - ], - )? - .into_any()) -} - /// Serialize maximum automatic-start safety evidence; `None` stays `None`. #[pyfunction] -fn serialize_maximum_start_safety<'py>(evidence: Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn serialize_maximum_start_safety<'py>( + evidence: Bound<'py, PyAny>, +) -> PyResult> { maximum_start_safety(&evidence) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(serialize_maximum_start_safety, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs index 61a5ab267c..1d5fe2e250 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs @@ -1,15 +1,11 @@ //! Reading planner values by attribute and building the JSON documents' dictionaries. -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList}; +use crate::plan_output::main::items::items; +use crate::plan_output::main::text::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyList; use sqlbuild_python_base::records::main::fields::field::field as native_field; -/// `str(value)`. -pub(crate) fn text(value: &Bound<'_, PyAny>) -> PyResult { - Ok(value.str()?.to_string()) -} - /// `str(value)`, or `None`. pub(crate) fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { if value.is_none() { @@ -23,11 +19,6 @@ pub(crate) fn texts(value: &Bound<'_, PyAny>) -> PyResult> { value.try_iter()?.map(|item| text(&item?)).collect() } -/// Every item of an iterable. -pub(crate) fn items<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { - value.try_iter()?.collect() -} - /// `value..value`, the value of an enum attribute. pub(crate) fn enum_value<'py>( value: &Bound<'py, PyAny>, @@ -44,26 +35,6 @@ pub(crate) fn optional_enum_value<'py>(value: &Bound<'py, PyAny>) -> PyResult( - py: Python<'py>, - entries: Vec<(&str, Bound<'py, PyAny>)>, -) -> PyResult> { - let built: Bound<'py, PyDict> = PyDict::new(py); - for (key, value) in entries { - built.set_item(key, value)?; - } - Ok(built) -} - -/// Any Rust value as a Python object. -pub(crate) fn any<'py, T: IntoPyObjectExt<'py>>( - py: Python<'py>, - value: T, -) -> PyResult> { - value.into_bound_py_any(py) -} - /// `list(value)`. pub(crate) fn list<'py>(value: &Bound<'py, PyAny>) -> PyResult> { Ok(PyList::new(value.py(), items(value)?)?.into_any()) diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/aligned_timestamp_bounds.rs b/crates/sqlbuild-python-planner/src/plan_output/main/aligned_timestamp_bounds.rs index c4f123e3c5..f6d4c0437a 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/aligned_timestamp_bounds.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/aligned_timestamp_bounds.rs @@ -1,16 +1,25 @@ //! Comparable bounds, presuming a naive side UTC when the other is aware. -use crate::plan_output::_helpers::display; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::plan_output::_helpers::display::as_datetime; +use crate::plan_output::_helpers::display::presume_utc; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Comparable bounds, presuming a naive side UTC when the other is aware. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, start, end))] pub fn aligned_timestamp_bounds<'py>( start: &Bound<'py, PyAny>, end: &Bound<'py, PyAny>, ) -> PyResult> { - display::aligned_timestamp_bounds(start, end) + let py: Python<'py> = start.py(); + let mut start_at: Bound<'py, PyAny> = as_datetime(start)?; + let mut end_at: Bound<'py, PyAny> = as_datetime(end)?; + if native_field(&start_at, "tzinfo")?.is_none() != native_field(&end_at, "tzinfo")?.is_none() { + start_at = presume_utc(start_at)?; + end_at = presume_utc(end_at)?; + } + PyTuple::new(py, [start_at, end_at]) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/any.rs b/crates/sqlbuild-python-planner/src/plan_output/main/any.rs index dcc624a50f..9788fce459 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/any.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/any.rs @@ -1,14 +1,9 @@ //! Any Rust value as a Python object. -use crate::plan_output::_helpers::values; use pyo3::IntoPyObjectExt; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; /// Any Rust value as a Python object. -#[inline] pub fn any<'py, T: IntoPyObjectExt<'py>>(py: Python<'py>, value: T) -> PyResult> { - values::any(py, value) + value.into_bound_py_any(py) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/cursor_bound_display.rs b/crates/sqlbuild-python-planner/src/plan_output/main/cursor_bound_display.rs index 8576e6d79b..efcca9a045 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/cursor_bound_display.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/cursor_bound_display.rs @@ -1,16 +1,34 @@ //! A cursor bound for display, collapsing a whole-day midnight timestamp to a date. -use crate::plan_output::_helpers::display; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::plan_output::_helpers::display::parsed_bound; +use crate::plan_output::_helpers::values::optional_text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_core::plan_display::constants::INTEGER_CURSOR; +use sqlbuild_core::plan_display::main::cursor_bound_display::cursor_bound_display as bound_display; +use sqlbuild_core::plan_display::models::ParsedBound; /// A cursor bound for display, collapsing a whole-day midnight timestamp to a date. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, value, cursor_type, cursor_grain))] pub fn cursor_bound_display( value: &Bound<'_, PyAny>, cursor_type: &Bound<'_, PyAny>, cursor_grain: &Bound<'_, PyAny>, ) -> PyResult { - display::cursor_bound_display(value, cursor_type, cursor_grain) + let cursor_type: Option = optional_text(cursor_type)?; + let raw: String = value.extract()?; + let parsed: Option = if cursor_type.as_deref() == Some(INTEGER_CURSOR) { + None + } else { + parsed_bound(value)? + }; + Ok(bound_display( + &raw, + ( + cursor_type.as_deref(), + optional_text(cursor_grain)?.as_deref(), + ), + parsed.as_ref(), + )) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/dict.rs b/crates/sqlbuild-python-planner/src/plan_output/main/dict.rs index 2ae1bc859c..a75df8c775 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/dict.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/dict.rs @@ -1,17 +1,16 @@ //! A dictionary of the entries, in order. -use crate::plan_output::_helpers::values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; /// A dictionary of the entries, in order. -#[inline] pub fn dict<'py>( py: Python<'py>, entries: Vec<(&str, Bound<'py, PyAny>)>, ) -> PyResult> { - values::dict(py, entries) + let built: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in entries { + built.set_item(key, value)?; + } + Ok(built) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/format_plan.rs b/crates/sqlbuild-python-planner/src/plan_output/main/format_plan.rs index f47ea03028..f2f4162f4a 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/format_plan.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/format_plan.rs @@ -1,13 +1,29 @@ //! The text plan grouped by reason with inline detail, optionally closed by its completion line. -use crate::plan_output::_helpers::plan_text; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::plan_output::_helpers::plan_text::PLAN_OPTIONS; +use crate::plan_output::_helpers::plan_text::report; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_core::cli_text::main::format_plan_completion::format_plan_completion; +use sqlbuild_core::cli_text::main::format_plan_text::format_plan_text; +use sqlbuild_core::cli_text::models::PlanTextReport; +use sqlbuild_python_base::boundary::models::KeywordArguments; /// The text plan grouped by reason with inline detail, optionally closed by its completion line. -#[inline] +#[pyfunction] +#[pyo3(signature = (**options))] pub fn format_plan(py: Python<'_>, options: Option>) -> PyResult { - plan_text::format_plan(py, options) + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "format_plan", options, PLAN_OPTIONS)?; + let report: PlanTextReport = report(&arguments)?; + let mut text: String = format_plan_text(&report); + if arguments.passed("changed_sections")? { + let sections: String = arguments.value("changed_sections")?.extract()?; + text = format!("{text}\n\n{sections}"); + } + if !arguments.flag("include_completion")? { + return Ok(text); + } + Ok(format!("{text}\n\n{}", format_plan_completion(&report))) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/format_plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/main/format_plan_json.rs index 90feb01094..c576d8b3a3 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/format_plan_json.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/format_plan_json.rs @@ -1,13 +1,119 @@ //! The plan as one JSON document, `json.dumps(indent=2)`. -use crate::plan_output::_helpers::plan_json; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; +use crate::plan_output::_helpers::plan_json::column_migrations_json; +use crate::plan_output::_helpers::plan_json::function_json; +use crate::plan_output::_helpers::plan_json::migrations_json; +use crate::plan_output::_helpers::plan_json::model_json; +use crate::plan_output::_helpers::plan_json::old_name_json; +use crate::plan_output::_helpers::plan_json::providers_json; +use crate::plan_output::_helpers::plan_json::python_entry_json; +use crate::plan_output::_helpers::plan_json::retention_json; +use crate::plan_output::_helpers::plan_json::seed_json; +use crate::plan_output::_helpers::plan_json::source_load_json; +use crate::plan_output::_helpers::plan_json::table_type_json; +use crate::plan_output::_helpers::plan_json::warning_json; +use crate::plan_output::_helpers::values::mapped; +use crate::plan_output::constants::CHANGED_KEY; +use crate::plan_output::main::any::any; +use crate::plan_output::main::dict::dict; +use crate::plan_output::main::items::items; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyList}; +use sqlbuild_core::plan_display::constants::SYNCED_COPY_KEY; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::compile_outputs::main::indented_json::indented_json; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The plan as one JSON document, `json.dumps(indent=2)`. -#[inline] +#[pyfunction] +#[pyo3(signature = (**options))] pub fn format_plan_json(py: Python<'_>, options: Option>) -> PyResult { - plan_json::format_plan_json(py, options) + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "format_plan_json", + options, + ( + &["plan"], + &["python_plan_entries", SYNCED_COPY_KEY, CHANGED_KEY], + ), + )?; + let plan: Bound<'_, PyAny> = arguments.value("plan")?; + let python_entries: Vec> = if arguments.passed("python_plan_entries")? { + items(&arguments.value("python_plan_entries")?)? + } else { + Vec::new() + }; + let length = |field: &str| -> PyResult { native_field(&plan, field)?.len() }; + let python_nodes: Vec> = python_entries + .iter() + .map(python_entry_json) + .collect::>()?; + let result: Bound<'_, PyDict> = dict( + py, + vec![ + ( + "selected_count", + any( + py, + length("model_entries")? + + length("seed_entries")? + + length("function_entries")?, + )?, + ), + ( + "source_load_count", + any(py, length("source_load_entries")?)?, + ), + ("python_node_count", any(py, python_nodes.len())?), + ( + "models", + mapped(&native_field(&plan, "model_entries")?, model_json)?, + ), + ( + "seeds", + mapped(&native_field(&plan, "seed_entries")?, seed_json)?, + ), + ( + "source_loads", + mapped( + &native_field(&plan, "source_load_entries")?, + source_load_json, + )?, + ), + ( + "functions", + mapped(&native_field(&plan, "function_entries")?, function_json)?, + ), + ("python_nodes", PyList::new(py, python_nodes)?.into_any()), + ("providers", providers_json(&plan, &python_entries)?), + ( + "warnings", + mapped(&native_field(&plan, "warnings")?, warning_json)?, + ), + ("retention", retention_json(&plan)?), + ("table_type_conversions", table_type_json(&plan)?), + ("migrations", migrations_json(&plan)?), + ("column_migrations", column_migrations_json(&plan)?), + ( + "old_names", + mapped( + &native_field(&plan, "old_name_view_entries")?, + old_name_json, + )?, + ), + ], + )?; + let metadata: Bound<'_, PyAny> = native_field(&plan, "metadata")?; + if metadata.is_truthy()? { + result.set_item("metadata", metadata)?; + } + let synced_copy: Bound<'_, PyAny> = arguments.value(SYNCED_COPY_KEY)?; + if !synced_copy.is_none() { + result.set_item(SYNCED_COPY_KEY, synced_copy)?; + } + if arguments.passed(CHANGED_KEY)? { + result.set_item(CHANGED_KEY, arguments.value(CHANGED_KEY)?)?; + } + indented_json(result.as_any()) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/future_cursor_safety.rs b/crates/sqlbuild-python-planner/src/plan_output/main/future_cursor_safety.rs index 7dbf12ecf7..f890726eae 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/future_cursor_safety.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/future_cursor_safety.rs @@ -1,12 +1,81 @@ //! Cap policy, bounds and physical input evidence of future-cursor safety, or `None`. -use crate::plan_output::_helpers::safety; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::plan_output::_helpers::safety::bounds; +use crate::plan_output::_helpers::safety::optional_render; +use crate::plan_output::_helpers::safety::render; +use crate::plan_output::_helpers::values::enum_value; +use crate::plan_output::main::dict::dict; +use crate::plan_output::main::items::items; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Cap policy, bounds and physical input evidence of future-cursor safety, or `None`. -#[inline] pub fn future_cursor_safety<'py>(evidence: &Bound<'py, PyAny>) -> PyResult> { - safety::future_cursor_safety(evidence) + let py: Python<'py> = evidence.py(); + if evidence.is_none() { + return Ok(evidence.clone()); + } + let relation: Bound<'py, PyAny> = native_field(evidence, "determining_relation")?; + let determining: Bound<'py, PyAny> = if relation.is_none() { + relation + } else { + dict( + py, + vec![ + ("relation", relation), + ( + "cursor_column", + native_field(evidence, "determining_cursor_column")?, + ), + ], + )? + .into_any() + }; + let mut inputs: Vec> = Vec::new(); + for item in items(&native_field(evidence, "inputs")?)? { + inputs.push(dict( + py, + vec![ + ("relation", native_field(&item, "relation")?), + ("cursor_column", native_field(&item, "cursor_column")?), + ("minimum", optional_render(native_field(&item, "minimum")?)?), + ("maximum", render(native_field(&item, "maximum")?)?), + ], + )?); + } + Ok(dict( + py, + vec![ + ("action", enum_value(evidence, "action")?), + ("max_distance", native_field(evidence, "max_distance")?), + ( + "invocation_time", + render(native_field(evidence, "invocation_time")?)?, + ), + ( + "discovered_bounds", + bounds(evidence, ("discovered_start", "discovered_end"))?, + ), + ( + "applied_bounds", + bounds(evidence, ("applied_start", "applied_end"))?, + ), + ( + "maximum_allowed_bounds", + bounds(evidence, ("maximum_allowed_start", "maximum_allowed_end"))?, + ), + ( + "future_start_detected", + native_field(evidence, "future_start_detected")?, + ), + ( + "future_end_detected", + native_field(evidence, "future_end_detected")?, + ), + ("determining_input", determining), + ("inputs", pyo3::types::PyList::new(py, inputs)?.into_any()), + ], + )? + .into_any()) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/inclusive_cursor_end.rs b/crates/sqlbuild-python-planner/src/plan_output/main/inclusive_cursor_end.rs index 359d9714ca..416956edfd 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/inclusive_cursor_end.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/inclusive_cursor_end.rs @@ -1,16 +1,23 @@ //! The last cursor value included by an exclusive end bound. -use crate::plan_output::_helpers::display; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::plan_output::_helpers::display::cursor_typing; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; +use sqlbuild_python_front::cursor::main::scalars::to_inclusive::to_inclusive; +use sqlbuild_python_front::cursor::main::scalars::try_parse::try_parse; /// The last cursor value included by an exclusive end bound. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, end, cursor_type, cursor_grain))] pub fn inclusive_cursor_end( end: &Bound<'_, PyAny>, cursor_type: &Bound<'_, PyAny>, cursor_grain: &Bound<'_, PyAny>, ) -> PyResult { - display::inclusive_cursor_end(end, cursor_type, cursor_grain) + let (effective, grain) = cursor_typing(cursor_type, cursor_grain)?; + let Some(parsed) = try_parse(end, &effective)? else { + return end.extract(); + }; + render_value(&to_inclusive(&parsed, &grain)?) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/items.rs b/crates/sqlbuild-python-planner/src/plan_output/main/items.rs index 269caa6c1f..beb0d4d41b 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/items.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/items.rs @@ -1,12 +1,8 @@ //! Every item of an iterable. -use crate::plan_output::_helpers::values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// Every item of an iterable. -#[inline] pub fn items<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { - values::items(value) + value.try_iter()?.collect() } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/maximum_start_safety.rs b/crates/sqlbuild-python-planner/src/plan_output/main/maximum_start_safety.rs index c84e933338..2144ec8b1c 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/maximum_start_safety.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/maximum_start_safety.rs @@ -1,12 +1,55 @@ //! Automatic-start policy and physical eligibility evidence, or `None`. -use crate::plan_output::_helpers::safety; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::plan_output::_helpers::safety::optional_render; +use crate::plan_output::_helpers::safety::render; +use crate::plan_output::_helpers::values::enum_value; +use crate::plan_output::main::dict::dict; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Automatic-start policy and physical eligibility evidence, or `None`. -#[inline] pub fn maximum_start_safety<'py>(evidence: &Bound<'py, PyAny>) -> PyResult> { - safety::maximum_start_safety(evidence) + let py: Python<'py> = evidence.py(); + if evidence.is_none() { + return Ok(evidence.clone()); + } + Ok(dict( + py, + vec![ + ("action", enum_value(evidence, "action")?), + ("max_ahead", native_field(evidence, "max_ahead")?), + ( + "invocation_time", + render(native_field(evidence, "invocation_time")?)?, + ), + ( + "physical_target_max", + render(native_field(evidence, "physical_target_max")?)?, + ), + ( + "highest_eligible_target_max", + optional_render(native_field(evidence, "highest_eligible_target_max")?)?, + ), + ( + "effective_start", + render(native_field(evidence, "effective_start")?)?, + ), + ( + "maximum_allowed_start", + render(native_field(evidence, "maximum_allowed_start")?)?, + ), + ( + "input", + dict( + py, + vec![ + ("relation", native_field(evidence, "target_relation")?), + ("cursor_column", native_field(evidence, "cursor_column")?), + ], + )? + .into_any(), + ), + ], + )? + .into_any()) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/optional_facts.rs b/crates/sqlbuild-python-planner/src/plan_output/main/optional_facts.rs index d5f5c7d1b4..2439cc7e29 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/optional_facts.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/optional_facts.rs @@ -1,13 +1,13 @@ //! Optional facts. -use crate::plan_output::_helpers::display; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::plan_output::_helpers::display::label_facts; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_core::plan_display::models::ModelLabelFacts; /// Optional facts. -#[inline] pub fn optional_facts(entry: &Bound<'_, PyAny>) -> PyResult> { - display::optional_facts(entry) + if entry.is_none() { + return Ok(None); + } + label_facts(entry).map(Some) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs b/crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs index 2971f3c847..63f14dcf35 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs @@ -1,12 +1,22 @@ //! Register the display functions and classes. -use crate::plan_output::_helpers::display; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::plan_output::_helpers::display::format_waiting_on_empty_inputs; +use crate::plan_output::main::aligned_timestamp_bounds::aligned_timestamp_bounds; +use crate::plan_output::main::cursor_bound_display::cursor_bound_display; +use crate::plan_output::main::inclusive_cursor_end::inclusive_cursor_end; +use crate::plan_output::main::resolve_effective_microbatch_batch_size::resolve_effective_microbatch_batch_size; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the display functions and classes. -#[inline] pub fn register_display(module: &Bound<'_, PyModule>) -> PyResult<()> { - display::register(module) + module.add_function(wrap_pyfunction!(inclusive_cursor_end, module)?)?; + module.add_function(wrap_pyfunction!(cursor_bound_display, module)?)?; + module.add_function(wrap_pyfunction!(format_waiting_on_empty_inputs, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_effective_microbatch_batch_size, + module + )?)?; + module.add_function(wrap_pyfunction!(aligned_timestamp_bounds, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/register_migrations.rs b/crates/sqlbuild-python-planner/src/plan_output/main/register_migrations.rs index 6f2e4c9382..8004dd6c69 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/register_migrations.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/register_migrations.rs @@ -1,12 +1,12 @@ //! Register the migrations functions and classes. -use crate::plan_output::_helpers::migrations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::plan_output::_helpers::migrations::format_pre_build; +use crate::plan_output::_helpers::migrations::render_old_name; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the migrations functions and classes. -#[inline] pub fn register_migrations(module: &Bound<'_, PyModule>) -> PyResult<()> { - migrations::register(module) + module.add_function(wrap_pyfunction!(format_pre_build, module)?)?; + module.add_function(wrap_pyfunction!(render_old_name, module)?) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_json.rs index a73f20e89e..1d8c833f94 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_json.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_json.rs @@ -1,12 +1,10 @@ //! Register the plan json functions and classes. -use crate::plan_output::_helpers::plan_json; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::plan_output::main::format_plan_json::format_plan_json; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the plan json functions and classes. -#[inline] pub fn register_plan_json(module: &Bound<'_, PyModule>) -> PyResult<()> { - plan_json::register(module) + module.add_function(wrap_pyfunction!(format_plan_json, module)?) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_text.rs index 63c7475ed1..52e691c6a4 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_text.rs @@ -1,12 +1,10 @@ //! Register the plan text functions and classes. -use crate::plan_output::_helpers::plan_text; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::plan_output::main::format_plan::format_plan; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the plan text functions and classes. -#[inline] pub fn register_plan_text(module: &Bound<'_, PyModule>) -> PyResult<()> { - plan_text::register(module) + module.add_function(wrap_pyfunction!(format_plan, module)?) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/register_safety.rs b/crates/sqlbuild-python-planner/src/plan_output/main/register_safety.rs index 9d82e7979d..b9fe38126e 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/register_safety.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/register_safety.rs @@ -1,12 +1,10 @@ //! Register the safety functions and classes. -use crate::plan_output::_helpers::safety; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::plan_output::_helpers::safety::serialize_maximum_start_safety; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the safety functions and classes. -#[inline] pub fn register_safety(module: &Bound<'_, PyModule>) -> PyResult<()> { - safety::register(module) + module.add_function(wrap_pyfunction!(serialize_maximum_start_safety, module)?) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/resolve_bounded_cursor_override.rs b/crates/sqlbuild-python-planner/src/plan_output/main/resolve_bounded_cursor_override.rs index 3f045b15c1..4dcc366b3e 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/resolve_bounded_cursor_override.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/resolve_bounded_cursor_override.rs @@ -1,22 +1,34 @@ //! A fully bounded inclusive operator override as an executable interval. -use crate::plan_output::_helpers::display; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::plan_output::_helpers::display::cursor_typing; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::main::scalars::to_exclusive::to_exclusive; +use sqlbuild_python_front::cursor::main::scalars::try_parse::try_parse; /// A fully bounded inclusive operator override as an executable interval. -#[inline] pub fn resolve_bounded_cursor_override<'py>( start_cursor_override: &Bound<'py, PyAny>, end_cursor_override: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, cursor_grain: &Bound<'py, PyAny>, ) -> PyResult>> { - display::resolve_bounded_cursor_override( - start_cursor_override, - end_cursor_override, - cursor_type, - cursor_grain, + let py: Python<'py> = start_cursor_override.py(); + if start_cursor_override.is_none() || end_cursor_override.is_none() { + return Ok(None); + } + let (effective, grain) = cursor_typing(cursor_type, cursor_grain)?; + let (Some(start), Some(end)) = ( + try_parse(start_cursor_override, &effective)?, + try_parse(end_cursor_override, &effective)?, + ) else { + return Ok(None); + }; + new_record_with( + py, + RecordId::CursorBounds, + &[("start", start), ("end", to_exclusive(&end, &grain)?)], ) + .map(Some) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/resolve_effective_microbatch_batch_size.rs b/crates/sqlbuild-python-planner/src/plan_output/main/resolve_effective_microbatch_batch_size.rs index aa542e4c8d..612c04153b 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/resolve_effective_microbatch_batch_size.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/resolve_effective_microbatch_batch_size.rs @@ -1,15 +1,24 @@ //! The explicit effective batch-size token resolved against the replay grain. -use crate::plan_output::_helpers::display; -use pyo3::prelude::PyErr; -use pyo3::prelude::Python; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{PyAnyMethods, PyErr, Python}; +use pyo3::pyfunction; +use sqlbuild_core::plan_display::main::effective_microbatch_batch_size::effective_microbatch_batch_size; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::types::RecordId; /// The explicit effective batch-size token resolved against the replay grain. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, batch_size, effective_grain))] pub fn resolve_effective_microbatch_batch_size( py: Python<'_>, batch_size: &str, effective_grain: &str, ) -> Result { - display::resolve_effective_microbatch_batch_size(py, batch_size, effective_grain) + effective_microbatch_batch_size(batch_size, effective_grain).map_err(|error| { + record_class(py, RecordId::CursorGrain) + .and_then(|grain| grain.call1((error.0,))) + .err() + .unwrap_or_else(|| PyValueError::new_err("unknown cursor grain")) + }) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/text.rs b/crates/sqlbuild-python-planner/src/plan_output/main/text.rs index fdc2e5954e..e92c227b48 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/text.rs @@ -1,12 +1,8 @@ //! `str(value)`. -use crate::plan_output::_helpers::values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// `str(value)`. -#[inline] pub fn text(value: &Bound<'_, PyAny>) -> PyResult { - values::text(value) + Ok(value.str()?.to_string()) } diff --git a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/actions.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/actions.rs index 2b433681a5..c8a3efa9b8 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/actions.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/actions.rs @@ -1,61 +1,4 @@ //! Per-model actions from each model's own change, and the incremental replacement predicate. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::planner_scope::constants::MODEL; - -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; - -const INCREMENTAL_MATERIALIZATION: &str = "incremental"; -const CREATE_TABLE_ACTION: &str = "create_table"; - -/// Each selected model's change with the backfill its own `replay_on_change` decided. -pub(crate) fn resolve_model_actions<'py>( - scope: &Bound<'py, PyAny>, - changes: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = scope.py(); - let scope = scope_data(scope)?; - let model_changes: Bound<'py, PyAny> = native_field(changes, "models")?; - let resolved: Bound<'py, PyDict> = PyDict::new(py); - for key in &scope.order { - if !scope.selected.contains(key) || key.0 != MODEL { - continue; - } - let name = &key.1; - let change: Bound<'py, PyAny> = mapping_get(&model_changes, name)?; - if !scope.models.contains_key(name) || change.is_none() { - continue; - } - let backfill: Bound<'py, PyAny> = native_field(&change, "backfill")?; - resolved.set_item( - name, - new_record_with( - py, - RecordId::ResolvedModelAction, - &[("change", change), ("backfill", backfill)], - )?, - )?; - } - new_record_with( - py, - RecordId::PlannerResolvedActions, - &[("models", resolved.into_any())], - ) -} - -/// Incremental replacements, whose bounds must ignore the destination maximum. -#[pyfunction] -#[pyo3(signature = (*, materialization_type, action))] -pub(crate) fn replaces_incremental_relation(materialization_type: &str, action: &str) -> bool { - materialization_type == INCREMENTAL_MATERIALIZATION && action == CREATE_TABLE_ACTION -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(replaces_incremental_relation, module)?) -} +pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +pub(crate) const CREATE_TABLE_ACTION: &str = "create_table"; diff --git a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs index 11fee40d95..d22a1fa85d 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs @@ -1,13 +1,14 @@ //! Detect how each selected model and function changed since its recorded build. +use crate::planner_changes::main::native_enum::native_enum; +use crate::planner_changes::main::planner_enum::planner_enum; use crate::planner_changes::models::ModelPass; use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::planner_changes::_helpers::results::native_enum; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_front::contracts::main::matching_dynamic_families::matching_dynamic_families; use sqlbuild_python_front::model_config::main::config::config_text::config_text; @@ -33,11 +34,9 @@ use sqlbuild_analysis::planner_changes::models::{ }; use crate::planner_changes::_helpers::renames::origin_names; -use crate::planner_changes::_helpers::results::backfill_result; -use crate::planner_changes::_helpers::results::change_kind; -use crate::planner_changes::_helpers::results::planner_enum; -use crate::planner_changes::_helpers::results::replay_policy_error; -use crate::planner_changes::_helpers::results::schema_findings; +use crate::planner_changes::_helpers::results::{ + backfill_result, change_kind, replay_policy_error, schema_findings, +}; use crate::planner_identity::main::compiled_function_fingerprint_sql::compiled_function_fingerprint_sql; use crate::planner_identity::main::function_local_hashes::function_local_hashes; use crate::planner_identity::main::model_declared_columns_hash::model_declared_columns_hash; @@ -66,7 +65,7 @@ const REPLAY_CONFIG_KEY: &str = "replay_on_change"; /// Selected model and function changes. #[pyfunction(name = "detect_changes")] #[pyo3(signature = (**options))] -fn detect_changes_of<'py>( +pub(crate) fn detect_changes_of<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -164,50 +163,6 @@ pub(crate) fn detect_changes<'py>( ) } -/// One model's change and backfill. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn detect_model_changes<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "detect_model_changes", - options, - ( - &[ - "model", - "snapshot", - "sql_analysis_enabled", - "query_change_tracking", - "full_refresh", - ], - &[ - "function_local_hashes", - "hook_version_hashes", - "expected_version_hash", - "expected_metadata_json", - ], - ), - )?; - let pass = ModelPass { - snapshot: arguments.value("snapshot")?, - query_change_tracking: arguments.flag("query_change_tracking")?, - full_refresh: arguments.flag("full_refresh")?, - function_local_hashes: arguments.value("function_local_hashes")?, - hook_version_hashes: arguments.value("hook_version_hashes")?, - }; - model_change( - &pass, - &arguments.value("model")?, - ( - arguments.value("expected_version_hash")?.extract()?, - arguments.value("expected_metadata_json")?.extract()?, - ), - ) -} - pub(crate) fn scope_model_changes<'py>( (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), query_change_tracking: Option, @@ -593,7 +548,7 @@ fn schema_findings_of<'py>( /// Schema findings between declared or inferred columns and warehouse columns. #[pyfunction(name = "detect_schema_changes")] #[pyo3(signature = (**options))] -fn schema_changes<'py>( +pub(crate) fn schema_changes<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -760,9 +715,3 @@ impl<'py> ChangeResult<'py> { ) } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(detect_changes_of, module)?)?; - module.add_function(wrap_pyfunction!(detect_model_changes, module)?)?; - module.add_function(wrap_pyfunction!(schema_changes, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs index 6946656b3c..cba6513864 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs @@ -1,11 +1,10 @@ //! Version-identity metadata helpers the remaining Python planner reads. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::planner_changes::main::version_identity_payload::version_identity_payload; use sqlbuild_analysis::planner_changes::main::with_declared_columns_hash::with_declared_columns_hash as with_hash; -use sqlbuild_analysis::planner_changes::main::without_declared_columns_hash::without_declared_columns_hash as without_hash; use sqlbuild_analysis::planner_changes::models::IdentityPayload; use sqlbuild_python_base::boundary::main::imported::imported; @@ -13,7 +12,7 @@ use sqlbuild_python_base::boundary::main::imported::imported; /// The decoded metadata that participates in version identity, the raw text when it is not JSON, or `None`. #[pyfunction] #[pyo3(signature = (metadata_json))] -fn version_identity_metadata_payload<'py>( +pub(crate) fn version_identity_metadata_payload<'py>( py: Python<'py>, metadata_json: Option<&str>, ) -> PyResult> { @@ -29,18 +28,9 @@ fn version_identity_metadata_payload<'py>( /// Fingerprint metadata JSON that also records the model's declared-columns hash. #[pyfunction] #[pyo3(signature = (*, metadata_json, declared_columns_hash))] -fn with_declared_columns_hash(metadata_json: &str, declared_columns_hash: &str) -> String { +pub(crate) fn with_declared_columns_hash( + metadata_json: &str, + declared_columns_hash: &str, +) -> String { with_hash(metadata_json, declared_columns_hash) } - -/// Fingerprint metadata JSON without the declared-columns hash key, if present. -#[pyfunction] -pub(crate) fn without_declared_columns_hash(metadata_json: &str) -> String { - without_hash(metadata_json) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(version_identity_metadata_payload, module)?)?; - module.add_function(wrap_pyfunction!(with_declared_columns_hash, module)?)?; - module.add_function(wrap_pyfunction!(without_declared_columns_hash, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs index 37a0ab4c90..dba4aa0cce 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs @@ -5,9 +5,9 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_project::fingerprints::main::compute_query_hash::compute_query_hash; use std::collections::BTreeSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_changes::main::origin_reference_names::origin_reference_names; use sqlbuild_analysis::planner_changes::models::{NameSpan, RenameCandidate}; @@ -123,7 +123,7 @@ fn utc<'py>(value: &Bound<'py, PyAny>) -> PyResult> { /// The rename mapping under which `query_sql` hashes to the recorded definition, or `None`. #[pyfunction(name = "origin_reference_names")] #[pyo3(signature = (*, query_sql, recorded, renames, dialect))] -fn origin_reference_names_of<'py>( +pub(crate) fn origin_reference_names_of<'py>( query_sql: String, recorded: &Bound<'py, PyAny>, renames: &Bound<'py, PyAny>, @@ -262,7 +262,3 @@ fn changed_referencing_query( } Ok(false) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(origin_reference_names_of, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/results.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/results.rs index 1e9fe96a0f..6a5d138a2c 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/results.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/results.rs @@ -1,5 +1,7 @@ //! The planner's Python change-detection result models. +use crate::planner_changes::main::native_enum::native_enum; +use crate::planner_changes::main::planner_enum::planner_enum; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyTuple; use pyo3::{IntoPyObject, PyErr}; @@ -10,7 +12,6 @@ use sqlbuild_analysis::planner_changes::models::{ Backfill, BackfillAction, ChangeKind, SchemaChangeKind, SchemaColumnSource, SchemaFindingRow, }; -use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; @@ -20,24 +21,6 @@ pub(crate) fn replay_policy_error(py: Python<'_>, error: UnknownReplayPolicy) -> new_plain_error(py, ErrorId::PlannerInput, &error.message()) } -/// `TYPES_MODULE.(value)`. -pub(crate) fn planner_enum<'py>( - py: Python<'py>, - name: RecordId, - value: &str, -) -> PyResult> { - new_record(py, name, (value,), None) -} - -/// A member of a natively declared planner enumeration, from its value. -pub(crate) fn native_enum<'py>( - py: Python<'py>, - name: RecordId, - value: &str, -) -> PyResult> { - new_record(py, name, (value,), None) -} - /// `BackfillResult(action=..., duration=...)`. pub(crate) fn backfill_result<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs index c12d5b8a4d..fee967c759 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs @@ -1,9 +1,7 @@ //! Selected model and function changes against expected version hashes and metadata JSON. use crate::planner_changes::_helpers::detect; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Selected model and function changes against expected version hashes and metadata JSON. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/detect_model_changes.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_model_changes.rs index 8a9928cb1a..cf5a66951f 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/detect_model_changes.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_model_changes.rs @@ -1,17 +1,52 @@ //! One model's change and backfill. -use crate::planner_changes::_helpers::detect; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::planner_changes::_helpers::detect::model_change; +use crate::planner_changes::models::ModelPass; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; /// One model's change and backfill. -#[inline] +#[pyfunction] +#[pyo3(signature = (**options))] pub fn detect_model_changes<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - detect::detect_model_changes(py, options) + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "detect_model_changes", + options, + ( + &[ + "model", + "snapshot", + "sql_analysis_enabled", + "query_change_tracking", + "full_refresh", + ], + &[ + "function_local_hashes", + "hook_version_hashes", + "expected_version_hash", + "expected_metadata_json", + ], + ), + )?; + let pass = ModelPass { + snapshot: arguments.value("snapshot")?, + query_change_tracking: arguments.flag("query_change_tracking")?, + full_refresh: arguments.flag("full_refresh")?, + function_local_hashes: arguments.value("function_local_hashes")?, + hook_version_hashes: arguments.value("hook_version_hashes")?, + }; + model_change( + &pass, + &arguments.value("model")?, + ( + arguments.value("expected_version_hash")?.extract()?, + arguments.value("expected_metadata_json")?.extract()?, + ), + ) } diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/model_change.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/model_change.rs index 37666185b0..f3640bac39 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/model_change.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/model_change.rs @@ -2,9 +2,7 @@ use crate::planner_changes::_helpers::detect; use crate::planner_changes::models::ModelPass; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Model change. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/native_enum.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/native_enum.rs index 6f1ed3130d..ed1872a423 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/native_enum.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/native_enum.rs @@ -1,18 +1,14 @@ //! A member of a natively declared planner enumeration, from its value. -use crate::planner_changes::_helpers::results; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::types::RecordId; /// A member of a natively declared planner enumeration, from its value. -#[inline] pub fn native_enum<'py>( py: Python<'py>, name: RecordId, value: &str, ) -> PyResult> { - results::native_enum(py, name, value) + new_record(py, name, (value,), None) } diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/plan_reference_renames.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/plan_reference_renames.rs index 4747e2cf64..27825bee37 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/plan_reference_renames.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/plan_reference_renames.rs @@ -1,11 +1,8 @@ //! This plan's renames plus recorded renames, read only when a selected query that references models changed. use crate::planner_changes::_helpers::renames; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; /// This plan's renames plus recorded renames, read only when a selected query that references models changed. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/planner_enum.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/planner_enum.rs index afdb8a753b..9f90be5077 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/planner_enum.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/planner_enum.rs @@ -1,18 +1,14 @@ //! `TYPES_MODULE.(value)`. -use crate::planner_changes::_helpers::results; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::types::RecordId; /// `TYPES_MODULE.(value)`. -#[inline] pub fn planner_enum<'py>( py: Python<'py>, name: RecordId, value: &str, ) -> PyResult> { - results::planner_enum(py, name, value) + new_record(py, name, (value,), None) } diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/register_actions.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/register_actions.rs index 20f5e62351..185515e8cf 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/register_actions.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/register_actions.rs @@ -1,12 +1,10 @@ //! Register the actions functions and classes. -use crate::planner_changes::_helpers::actions; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_changes::main::replaces_incremental_relation::replaces_incremental_relation; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the actions functions and classes. -#[inline] pub fn register_actions(module: &Bound<'_, PyModule>) -> PyResult<()> { - actions::register(module) + module.add_function(wrap_pyfunction!(replaces_incremental_relation, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs index a2aa945ca0..9a13e5a439 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs @@ -1,12 +1,14 @@ //! Register the detect functions and classes. -use crate::planner_changes::_helpers::detect; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_changes::_helpers::detect::detect_changes_of; +use crate::planner_changes::_helpers::detect::schema_changes; +use crate::planner_changes::main::detect_model_changes::detect_model_changes; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the detect functions and classes. -#[inline] pub fn register_detect(module: &Bound<'_, PyModule>) -> PyResult<()> { - detect::register(module) + module.add_function(wrap_pyfunction!(detect_changes_of, module)?)?; + module.add_function(wrap_pyfunction!(detect_model_changes, module)?)?; + module.add_function(wrap_pyfunction!(schema_changes, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs index 46b94c628a..abacb50745 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs @@ -1,12 +1,14 @@ //! Register the metadata functions and classes. -use crate::planner_changes::_helpers::metadata; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_changes::_helpers::metadata::version_identity_metadata_payload; +use crate::planner_changes::_helpers::metadata::with_declared_columns_hash; +use crate::planner_changes::main::without_declared_columns_hash::without_declared_columns_hash; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the metadata functions and classes. -#[inline] pub fn register_metadata(module: &Bound<'_, PyModule>) -> PyResult<()> { - metadata::register(module) + module.add_function(wrap_pyfunction!(version_identity_metadata_payload, module)?)?; + module.add_function(wrap_pyfunction!(with_declared_columns_hash, module)?)?; + module.add_function(wrap_pyfunction!(without_declared_columns_hash, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/register_renames.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/register_renames.rs index d008a65d5b..d267b7a6b9 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/register_renames.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/register_renames.rs @@ -1,12 +1,10 @@ //! Register the renames functions and classes. -use crate::planner_changes::_helpers::renames; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_changes::_helpers::renames::origin_reference_names_of; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the renames functions and classes. -#[inline] pub fn register_renames(module: &Bound<'_, PyModule>) -> PyResult<()> { - renames::register(module) + module.add_function(wrap_pyfunction!(origin_reference_names_of, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/replaces_incremental_relation.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/replaces_incremental_relation.rs index 373ae416f2..68df46256e 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/replaces_incremental_relation.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/replaces_incremental_relation.rs @@ -1,9 +1,12 @@ //! Incremental replacements, whose bounds must ignore the destination maximum. -use crate::planner_changes::_helpers::actions; +use crate::planner_changes::_helpers::actions::CREATE_TABLE_ACTION; +use crate::planner_changes::_helpers::actions::INCREMENTAL_MATERIALIZATION; +use pyo3::pyfunction; /// Incremental replacements, whose bounds must ignore the destination maximum. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, materialization_type, action))] pub fn replaces_incremental_relation(materialization_type: &str, action: &str) -> bool { - actions::replaces_incremental_relation(materialization_type, action) + materialization_type == INCREMENTAL_MATERIALIZATION && action == CREATE_TABLE_ACTION } diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/resolve_model_actions.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/resolve_model_actions.rs index 94c44cdeaa..ff5cfa8958 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/resolve_model_actions.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/resolve_model_actions.rs @@ -1,15 +1,45 @@ //! Each selected model's change with the backfill its own `replay_on_change` decided. -use crate::planner_changes::_helpers::actions; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::planner_scope::constants::MODEL; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; /// Each selected model's change with the backfill its own `replay_on_change` decided. -#[inline] pub fn resolve_model_actions<'py>( scope: &Bound<'py, PyAny>, changes: &Bound<'py, PyAny>, ) -> PyResult> { - actions::resolve_model_actions(scope, changes) + let py: Python<'py> = scope.py(); + let scope = scope_data(scope)?; + let model_changes: Bound<'py, PyAny> = native_field(changes, "models")?; + let resolved: Bound<'py, PyDict> = PyDict::new(py); + for key in &scope.order { + if !scope.selected.contains(key) || key.0 != MODEL { + continue; + } + let name = &key.1; + let change: Bound<'py, PyAny> = mapping_get(&model_changes, name)?; + if !scope.models.contains_key(name) || change.is_none() { + continue; + } + let backfill: Bound<'py, PyAny> = native_field(&change, "backfill")?; + resolved.set_item( + name, + new_record_with( + py, + RecordId::ResolvedModelAction, + &[("change", change), ("backfill", backfill)], + )?, + )?; + } + new_record_with( + py, + RecordId::PlannerResolvedActions, + &[("models", resolved.into_any())], + ) } diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/scope_model_changes.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/scope_model_changes.rs index b50f8bb6de..82e98e9045 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/scope_model_changes.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/scope_model_changes.rs @@ -1,9 +1,7 @@ //! Scope model changes. use crate::planner_changes::_helpers::detect; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; /// Scope model changes. diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/without_declared_columns_hash.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/without_declared_columns_hash.rs index 7bb7da6e35..4e60201ec8 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/without_declared_columns_hash.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/without_declared_columns_hash.rs @@ -1,9 +1,10 @@ //! Fingerprint metadata JSON without the declared-columns hash key, if present. -use crate::planner_changes::_helpers::metadata; +use pyo3::pyfunction; +use sqlbuild_analysis::planner_changes::main::without_declared_columns_hash::without_declared_columns_hash as without_hash; /// Fingerprint metadata JSON without the declared-columns hash key, if present. -#[inline] +#[pyfunction] pub fn without_declared_columns_hash(metadata_json: &str) -> String { - metadata::without_declared_columns_hash(metadata_json) + without_hash(metadata_json) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/contexts.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/contexts.rs index 135d35aeae..601e006dbb 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/contexts.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/contexts.rs @@ -1,15 +1,14 @@ //! Relation, model-plan and entry-build contexts: built from keyword arguments and read by plan entry construction. +use crate::planner_entries::main::entries::default_microbatch_limit_action::default_microbatch_limit_action; use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; use pyo3::types::{PyDict, PyFrozenSet}; -use crate::planner_entries::models::ModelPlanContext; -use crate::planner_entries::models::PlanEntryBuildInputs; -use crate::planner_entries::models::PlannerRelationsContext; +use crate::planner_entries::models::{ + ModelPlanContext, PlanEntryBuildInputs, PlannerRelationsContext, +}; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::types::RecordId; /// The argument `name`, or `default()` when it was not passed. fn value_or<'py>( @@ -164,8 +163,3 @@ impl PlanEntryBuildInputs { }) } } - -/// The microbatch limit action used when a project configures none. -pub(crate) fn default_microbatch_limit_action(py: Python<'_>) -> PyResult> { - enum_member(py, RecordId::MicrobatchLimitAction, "ERROR") -} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs index 589c6e3b93..c98bdf79f5 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs @@ -3,9 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::planner_entries::models::ModelPlanContext; -use crate::planner_entries::models::PlanEntryBuildInputs; -use crate::planner_entries::models::PlannerRelationsContext; +use crate::planner_entries::models::{ + ModelPlanContext, PlanEntryBuildInputs, PlannerRelationsContext, +}; pub(crate) fn read_model_plan_context<'py>( value: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs index 907c9bfc8a..49b574f80d 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs @@ -1,18 +1,12 @@ //! Per-model plan entries from resolved change results and warehouse snapshot facts. -use crate::planner_changes::main::detect_model_changes::detect_model_changes; use crate::planner_changes::main::replaces_incremental_relation::replaces_incremental_relation; -use crate::planner_entries::_helpers::entries::values::MICROBATCH_CONCURRENCY_MESSAGE; -use crate::planner_entries::_helpers::entries::values::error_of; -use crate::planner_entries::_helpers::entries::values::native; -use crate::planner_entries::_helpers::entries::values::none; -use crate::planner_entries::_helpers::entries::values::planner_type; -use crate::planner_entries::_helpers::entries::values::replaced; -use crate::planner_entries::_helpers::entries::values::string; -use crate::planner_entries::models::ModelPlanContext; -use crate::planner_entries::models::PlanEntriesRequest; -use crate::planner_entries::models::PlanEntryBuildInputs; -use crate::planner_entries::models::PlannerRelationsContext; +use crate::planner_entries::_helpers::entries::values::{ + MICROBATCH_CONCURRENCY_MESSAGE, error_of, native, none, planner_type, replaced, string, +}; +use crate::planner_entries::models::{ + ModelPlanContext, PlanEntriesRequest, PlanEntryBuildInputs, PlannerRelationsContext, +}; use crate::planner_phases::main::inputs::class_field::class_field; use crate::planner_phases::main::rebuilds::check_full_rebuild_allowed::check_full_rebuild_allowed; use crate::planner_phases::main::rebuilds::describe_full_rebuild_cause::describe_full_rebuild_cause; @@ -39,11 +33,10 @@ use crate::planner_strategy::main::model_entries::assemble_model_plan_entry::ass use crate::planner_strategy::main::strategy::materialization_of::materialization_of; use crate::planner_strategy::main::strategy::resolve_model_plan_action::resolve_model_plan_action; use crate::planner_strategy::main::values::text::text; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; @@ -58,7 +51,7 @@ use sqlbuild_python_project::semantic_shapes::main::source_binding_diagnostics:: const MICROBATCH_COMPLEXITY_CODE: &str = "MICROBATCH_CONCURRENCY_COMPLEXITY"; /// The `ModelPlanContext` fields every model's planning reads, read once per plan. -struct ContextReads<'py> { +pub(crate) struct ContextReads<'py> { context: Bound<'py, PyAny>, lookups: CursorInputLookups<'py>, resolution: Resolution<'py>, @@ -66,7 +59,7 @@ struct ContextReads<'py> { } impl<'py> ContextReads<'py> { - fn of( + pub(crate) fn of( context: Bound<'py, PyAny>, adapter: Bound<'py, PyAny>, external_resolver: Bound<'py, PyAny>, @@ -92,19 +85,19 @@ impl<'py> ContextReads<'py> { } /// One model's planning request after change detection. -struct ModelPlanRequest<'a, 'py> { - model: Bound<'py, PyAny>, - snapshot: Bound<'py, PyAny>, - context: &'a ContextReads<'py>, - sql_analysis_enabled: bool, - full_refresh: bool, - overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), - change_result: Bound<'py, PyAny>, - protect_full_refresh: bool, +pub(crate) struct ModelPlanRequest<'a, 'py> { + pub(crate) model: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) context: &'a ContextReads<'py>, + pub(crate) sql_analysis_enabled: bool, + pub(crate) full_refresh: bool, + pub(crate) overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) change_result: Bound<'py, PyAny>, + pub(crate) protect_full_refresh: bool, } /// Build a model plan entry and its warnings from a resolved change result. -fn plan_model_from_change<'py>( +pub(crate) fn plan_model_from_change<'py>( request: &ModelPlanRequest<'_, 'py>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let model = &request.model; @@ -546,69 +539,6 @@ pub(crate) fn plan_entries<'py>( Ok(ModelEntryResults { entries, warnings }) } -/// Detect changes and build a plan entry and warnings for a single model. -#[pyfunction] -#[pyo3(signature = (**kwargs))] -pub(crate) fn plan_model<'py>( - py: Python<'py>, - kwargs: Option>, -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - let args = KeywordArguments::read( - py, - "plan_model", - kwargs, - ( - &[ - "model", - "snapshot", - "adapter", - "context", - "sql_analysis_enabled", - "query_change_tracking", - "full_refresh", - "cursor_overrides", - ], - &["external_sql_reference_resolver"], - ), - )?; - let model = args.value("model")?; - let snapshot = args.value("snapshot")?; - let change_result = detect_model_changes( - py, - Some(keywords( - py, - &[ - ("model", model.clone()), - ("snapshot", snapshot.clone()), - ("sql_analysis_enabled", args.value("sql_analysis_enabled")?), - ( - "query_change_tracking", - args.value("query_change_tracking")?, - ), - ("full_refresh", args.value("full_refresh")?), - ], - )?), - )?; - let overrides = args.value("cursor_overrides")?; - plan_model_from_change(&ModelPlanRequest { - model, - snapshot, - context: &ContextReads::of( - args.value("context")?, - args.value("adapter")?, - args.value("external_sql_reference_resolver")?, - )?, - sql_analysis_enabled: args.flag("sql_analysis_enabled")?, - full_refresh: args.flag("full_refresh")?, - overrides: ( - native_field(&overrides, "start_cursor_override")?, - native_field(&overrides, "end_cursor_override")?, - ), - change_result, - protect_full_refresh: true, - }) -} - /// Check if any scope dependency is in the selected keys. #[pyfunction] #[pyo3(signature = (*, scope_deps, selected_keys))] @@ -623,8 +553,3 @@ pub(crate) fn scope_overlaps( } Ok(false) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(plan_model, module)?)?; - module.add_function(wrap_pyfunction!(scope_overlaps, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs index 396627db3a..e6525f08cd 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs @@ -1,8 +1,7 @@ //! Relation locations and source metadata for plan entry construction. use crate::planner_entries::_helpers::entries::values::planner_model; -use crate::planner_entries::models::PlannerRelationsContext; -use crate::planner_entries::models::RelationsRequest; +use crate::planner_entries::models::{PlannerRelationsContext, RelationsRequest}; use crate::planner_resolution::main::references::apply_deferred_locations::apply_deferred_locations; use crate::planner_resolution::main::references::build_function_locations::build_function_locations; use crate::planner_resolution::main::references::build_model_locations::build_model_locations; @@ -12,9 +11,9 @@ use crate::planner_warehouse::main::sources::build_source_read_map::build_source use crate::planner_warehouse::main::sources::gather_source_inspection::gather_source_inspection; use crate::planner_warehouse::main::sources::semantic_source_columns::semantic_source_columns; use crate::planner_warehouse::main::sources::with_declared_source_reads::with_declared_source_reads; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -218,7 +217,7 @@ pub(crate) fn planner_relations_context<'py>( /// Resolve relation locations and source metadata for plan entry construction. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn build_planner_relations_context<'py>( +pub(crate) fn build_planner_relations_context<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -249,7 +248,3 @@ fn build_planner_relations_context<'py>( relation_keys: args.value("relation_keys")?, }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_planner_relations_context, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs index 241605c074..37cfc3b0c6 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs @@ -1,9 +1,8 @@ //! Clone planning: destination and origin entries, and deferred-clone graph traversal. -use crate::planner_entries::_helpers::entries::values::native; -use crate::planner_entries::_helpers::entries::values::planner_model; -use crate::planner_entries::_helpers::entries::values::planner_type; -use crate::planner_entries::_helpers::entries::values::string; +use crate::planner_entries::_helpers::entries::values::{ + native, planner_model, planner_type, string, +}; use crate::planner_entries::models::ModelPlanContext; use crate::planner_entries::types::PhaseTimings; use crate::planner_identity::main::build_compiled_function_fingerprint_sql::build_compiled_function_fingerprint_sql; @@ -17,9 +16,9 @@ use crate::planner_resolution::models::Resolution; use crate::planner_strategy::main::strategy::materialization_of::materialization_of; use crate::planner_warehouse::main::sources::gather_source_columns::gather_source_columns; use pyo3::PyRef; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::graph::main::downstream_edges::downstream_edges; use sqlbuild_core::graph_walk::main::clone_ancestors::clone_ancestors; use sqlbuild_core::graph_walk::models::ClonePolicy; @@ -755,8 +754,3 @@ pub(crate) fn resolve_skipped_view_chain<'py>( )?; CloneGraph::read(&args)?.walk(true) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_clone_boundary, module)?)?; - module.add_function(wrap_pyfunction!(resolve_skipped_view_chain, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs index 3d477ad341..88473cfaf3 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs @@ -1,25 +1,20 @@ //! Static plans for commands that do not build models, and direct seed identity reads. -use crate::planner_entries::_helpers::entries::relations::planner_relations_context; -use crate::planner_entries::_helpers::entries::values::boolean; -use crate::planner_entries::_helpers::entries::values::none; -use crate::planner_entries::_helpers::entries::values::planner_model; -use crate::planner_entries::_helpers::entries::values::planner_type; -use crate::planner_entries::_helpers::plans::plan_output::function_entry; -use crate::planner_entries::_helpers::plans::plan_output::selected_test_entries; -use crate::planner_entries::_helpers::plans::plan_output::test_selected; -use crate::planner_entries::models::RelationsRequest; +use crate::planner_entries::_helpers::entries::values::{ + boolean, none, planner_model, planner_type, +}; +use crate::planner_entries::_helpers::plans::plan_output::{ + function_entry, selected_test_entries, test_selected, +}; use crate::planner_identity::main::build_compiled_function_fingerprint_sql::build_compiled_function_fingerprint_sql; use crate::planner_identity::main::build_seed_identity::build_seed_identity; -use crate::planner_strategy::main::audit_entries::build_selected_audit_entries::build_selected_audit_entries; use crate::planner_strategy::main::model_entries::extract_seed_columns::extract_seed_columns; -use crate::planner_strategy::main::strategy::materialization_of::materialization_of; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{ PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PySet, PySetMethods, PyString, PyTuple, }; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_SEED; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -27,13 +22,10 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; use sqlbuild_python_project::planner_phases::main::object::object; -use sqlbuild_python_project::planner_scope::main::resolve_planner_scopes::resolve_planner_scopes; use sqlbuild_python_project::resource_views::main::function_reads::deps::deps; use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; @@ -41,12 +33,12 @@ use std::collections::{BTreeMap, BTreeSet}; /// Optional fields of a static command plan; absent ones take `PlanOutput` defaults. #[derive(Default)] -struct StaticPlanParts<'py> { - execution_order: Option>, - fields: Vec<(&'static str, Bound<'py, PyAny>)>, +pub(crate) struct StaticPlanParts<'py> { + pub(crate) execution_order: Option>, + pub(crate) fields: Vec<(&'static str, Bound<'py, PyAny>)>, } -fn base_plan_output<'py>( +pub(crate) fn base_plan_output<'py>( project: &Bound<'py, PyAny>, scope: &Bound<'py, PyAny>, relations: &Bound<'py, PyAny>, @@ -100,106 +92,6 @@ fn base_plan_output<'py>( planner_model(py, RecordId::PlanOutput, &fields) } -/// Resolve command selection through the canonical build selector phase. -#[pyfunction] -#[pyo3(signature = (*, project, selection, auto_load_sources = false))] -pub(crate) fn resolve_static_command_scope<'py>( - project: &Bound<'py, PyAny>, - selection: &Bound<'py, PyAny>, - auto_load_sources: bool, -) -> PyResult> { - let py = project.py(); - resolve_planner_scopes( - project, - selection, - &planner_model( - py, - RecordId::PlannerPolicies, - &[("auto_load_sources", boolean(py, auto_load_sources)?)], - )?, - )? - .getattr("selected_scope") -} - -/// Resolve canonical locations and source routing without warehouse inventory. -#[pyfunction] -#[pyo3(signature = (**kwargs))] -pub(crate) fn resolve_static_relation_context<'py>( - py: Python<'py>, - kwargs: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "resolve_static_relation_context", - kwargs, - ( - &["project", "adapter", "scope"], - &[ - "deferral", - "project_config", - "local_config", - "relation_keys", - ], - ), - )?; - planner_relations_context(RelationsRequest { - project: args.value("project")?, - adapter: args.value("adapter")?, - connection: none(py), - scope: args.value("scope")?, - deferral: args.value("deferral")?, - project_config: args.value("project_config")?, - local_config: args.value("local_config")?, - known_source_columns: PyDict::new(py).into_any(), - relation_keys: args.value("relation_keys")?, - }) -} - -/// Project selected audits from canonical static state. -#[pyfunction] -#[pyo3(signature = (*, project, adapter, scope, relations))] -pub(crate) fn build_audit_command_plan<'py>( - project: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, - scope: &Bound<'py, PyAny>, - relations: &Bound<'py, PyAny>, -) -> PyResult> { - let py = project.py(); - let materializations = PyDict::new(py); - for model in compiled_models(project)?.try_iter()? { - let model = model?; - materializations.set_item( - name_object(&model)?, - native_field(&materialization_of(&model)?, "value")?, - )?; - } - let audits = object( - py, - build_selected_audit_entries(Some(&keywords( - py, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("scope", scope.clone()), - ("relations", relations.clone()), - ("model_materializations", materializations.into_any()), - ], - )?))?, - )?; - base_plan_output( - project, - scope, - relations, - StaticPlanParts { - execution_order: None, - fields: vec![( - "audit_entries", - PyTuple::new(py, audits.try_iter()?.collect::>>()?)?.into_any(), - )], - }, - ) -} - /// The tests' direct function dependencies and every project function those call. fn called_function_keys<'py>( project: &Bound<'py, PyAny>, @@ -403,15 +295,6 @@ pub(crate) fn build_seed_command_plan<'py>( ) } -/// Project static relation data for Python checks. -pub(crate) fn build_relation_command_plan<'py>( - project: &Bound<'py, PyAny>, - scope: &Bound<'py, PyAny>, - relations: &Bound<'py, PyAny>, -) -> PyResult> { - base_plan_output(project, scope, relations, StaticPlanParts::default()) -} - /// A schema target's sort key: the database with `None` first as an empty name, then the schema. type Target = (Option, String); @@ -514,12 +397,3 @@ pub(crate) fn read_selected_seed_fingerprints<'py>( } Ok(fingerprints) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_static_command_scope, module)?)?; - module.add_function(wrap_pyfunction!(resolve_static_relation_context, module)?)?; - module.add_function(wrap_pyfunction!(build_audit_command_plan, module)?)?; - module.add_function(wrap_pyfunction!(build_test_command_plan, module)?)?; - module.add_function(wrap_pyfunction!(build_seed_command_plan, module)?)?; - module.add_function(wrap_pyfunction!(read_selected_seed_fingerprints, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs index 119d103918..4a069d1b96 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs @@ -1,17 +1,11 @@ //! Display-only plans for external plan surfaces when warehouse planning cannot resolve. -use crate::planner_entries::_helpers::entries::values::boolean; -use crate::planner_entries::_helpers::entries::values::native; -use crate::planner_entries::_helpers::entries::values::none; -use crate::planner_entries::_helpers::entries::values::planner_model; -use crate::planner_entries::_helpers::entries::values::planner_type; -use crate::planner_entries::_helpers::entries::values::string; +use crate::planner_entries::_helpers::entries::values::{ + boolean, native, none, planner_model, planner_type, string, +}; use crate::planner_phases::main::rebuilds::resolve_model_full_refresh::resolve_model_full_refresh; use crate::planner_strategy::main::strategy::materialization_of::materialization_of; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::graph::main::downstream_edges::downstream_edges; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; @@ -19,10 +13,6 @@ use sqlbuild_python_front::model_config::main::config::model_values::model_value use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; -use sqlbuild_python_project::graph::main::project::edge_dict::edge_dict; -use sqlbuild_python_project::graph::models::KeyCache; -use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; use sqlbuild_python_project::planner_phases::main::object::object; const VIEW: &str = "view"; @@ -54,7 +44,7 @@ fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult( +pub(crate) fn display_entry<'py>( model: &Bound<'py, PyAny>, full_refresh: bool, ) -> PyResult> { @@ -132,50 +122,3 @@ fn display_entry<'py>( ], ) } - -/// Build a display-only plan when warehouse planning cannot fully resolve. -#[pyfunction] -#[pyo3(signature = (*, project, selected_model_names, full_refresh))] -pub(crate) fn build_display_only_sqlbuild_plan<'py>( - project: &Bound<'py, PyAny>, - selected_model_names: &Bound<'py, PyAny>, - full_refresh: bool, -) -> PyResult> { - let py: Python<'py> = project.py(); - let selected = PyFrozenSet::new( - py, - selected_model_names - .try_iter()? - .collect::>>()?, - )?; - let mut entries: Vec> = Vec::new(); - for model in compiled_models(project)?.try_iter()? { - let model = model?; - if selected.contains(name_object(&model)?)? { - entries.push(display_entry(&model, full_refresh)?); - } - } - let keys: Vec> = entries - .iter() - .map(|entry| native_field(entry, "key")) - .collect::>()?; - let upstream = execution_deps(project)?; - let cache = KeyCache::new(py)?; - let downstream = edge_dict(&cache, &downstream_edges(&upstream))?.into_any(); - let upstream = edge_dict(&cache, &upstream)?.into_any(); - planner_model( - py, - RecordId::PlanOutput, - &[ - ("execution_order", PyTuple::new(py, &keys)?.into_any()), - ("model_entries", PyTuple::new(py, entries)?.into_any()), - ("selected_keys", PyFrozenSet::new(py, keys)?.into_any()), - ("upstream_deps", upstream), - ("downstream_deps", downstream), - ], - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_display_only_sqlbuild_plan, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs index d7f6fcfd7d..1be05282ab 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs @@ -5,11 +5,8 @@ use sqlbuild_host::entries::models::AdapterHost; use crate::planner_changes::main::detect_changes::detect_changes; use crate::planner_changes::main::resolve_model_actions::resolve_model_actions; -use crate::planner_entries::_helpers::entries::values::native; -use crate::planner_entries::_helpers::entries::values::replaced; -use crate::planner_entries::_helpers::plans::test_planning::PlanningRequest; +use crate::planner_entries::_helpers::entries::values::{native, replaced}; use crate::planner_entries::_helpers::plans::test_planning::SqlTestPlanning; -use crate::planner_entries::_helpers::plans::test_planning::start_planning; use crate::planner_phases::main::inputs::class_field::class_field; use crate::planner_phases::main::inputs::gather_planner_warehouse_state::gather_planner_warehouse_state; use crate::planner_phases::main::inputs::warehouse_field::warehouse_field; @@ -25,22 +22,15 @@ use crate::planner_phases::main::outputs::unselected_upstream_models::unselected use crate::planner_phases::main::rebuilds::build_planner_entry_results::build_planner_entry_results; use crate::planner_phases::main::rebuilds::defer_to_migration_origins::defer_to_migration_origins; use crate::planner_phases::main::rebuilds::reconcile_execution_changes::reconcile_execution_changes; -use crate::planner_phases::models::PlanOutputContext; -use crate::planner_phases::models::PlanWarningContext; -use crate::planner_phases::models::PlannedModelEntries; use crate::planner_phases::models::PlannerIdentityContext; -use crate::planner_phases::models::PlannerRuntime; -use crate::planner_phases::models::StoragePolicyContext; +use crate::planner_phases::models::{ + PlanOutputContext, PlanWarningContext, PlannedModelEntries, StoragePolicyContext, +}; use crate::planner_resolution::main::cursor::type_cursor_overrides::type_cursor_overrides; use crate::planner_warehouse::main::sources::build_planner_source_freshness_result::build_planner_source_freshness_result; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::build_command::main::utc_now::utc_now; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_project::planner_scope::main::resolve_planner_scopes::resolve_planner_scopes; @@ -48,9 +38,9 @@ use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use std::time::Instant; -const INSPECTION_CATALOG: AdapterHost = AdapterHost::OpenInspectionCatalog; +pub(crate) const INSPECTION_CATALOG: AdapterHost = AdapterHost::OpenInspectionCatalog; /// Selected scopes, and the overrides with cursor overrides typed for the selected models. -fn scopes_and_cursor_overrides<'py>( +pub(crate) fn scopes_and_cursor_overrides<'py>( args: &KeywordArguments<'py>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let project = args.value("project")?; @@ -142,7 +132,7 @@ fn deferred_migration_origins<'py>( } /// The plan phases after warehouse inspection, run inside the inspection catalog. -fn planned_output<'py>( +pub(crate) fn planned_output<'py>( args: &KeywordArguments<'py>, runtime: &Bound<'py, PyAny>, (scopes, overrides): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -235,75 +225,3 @@ fn planned_output<'py>( } Ok(plan_output) } - -/// Build the execution plan for one selection against live warehouse state. -#[pyfunction] -#[pyo3(signature = (**kwargs))] -pub(crate) fn build_execution_plan<'py>( - py: Python<'py>, - kwargs: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "build_execution_plan", - kwargs, - ( - &[ - "project", - "adapter", - "connection", - "selection", - "overrides", - "deferral", - "policies", - ], - &["on_progress", "project_config", "local_config"], - ), - )?; - let project = args.value("project")?; - let adapter = args.value("adapter")?; - let connection = args.value("connection")?; - let runtime = Bound::new( - py, - PlannerRuntime { - project: project.clone().unbind(), - adapter: adapter.clone().unbind(), - connection: connection.clone().unbind(), - project_config: args.value("project_config")?.unbind(), - local_config: args.value("local_config")?.unbind(), - on_progress: args.value("on_progress")?.unbind(), - invocation_time: utc_now(py)?.unbind(), - }, - )? - .into_any(); - let (scopes, overrides) = scopes_and_cursor_overrides(&args)?; - let inspection = class_field(&scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?; - let test_planning = start_planning( - py, - PlanningRequest { - project: project.clone().unbind(), - adapter: adapter.clone().unbind(), - selected_keys: native_field(&inspection, "selected_keys")?.unbind(), - sql_test_selection: native_field(&inspection, "sql_test_selection")?.unbind(), - }, - native_field(&args.value("policies")?, "plan_sql_tests")?.is_truthy()?, - )?; - let planned = (|| { - let catalog = call_with( - &host_entry(py, INSPECTION_CATALOG)?, - &[("adapter", adapter), ("connection", connection)], - )?; - within(&catalog, |_| { - planned_output(&args, &runtime, (&scopes, &overrides), test_planning.get()) - }) - })(); - let joined = test_planning.get().result(py); - let planned = planned?; - joined?; - Ok(planned) -} - -/// Add the execution plan entry point. -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_execution_plan, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs index 254694485f..327003a020 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs @@ -1,19 +1,16 @@ //! `PlanOutput` assembly from planned model entries, seeds, sources, functions, audits and tests. use crate::planner_entries::_helpers::entries::model_entries::scope_overlaps; -use crate::planner_entries::_helpers::entries::values::error_of; -use crate::planner_entries::_helpers::entries::values::native; -use crate::planner_entries::_helpers::entries::values::none; -use crate::planner_entries::_helpers::entries::values::planner_model; -use crate::planner_entries::_helpers::entries::values::planner_type; -use crate::planner_entries::_helpers::entries::values::string; +use crate::planner_entries::_helpers::entries::values::{ + error_of, native, none, planner_model, planner_type, string, +}; use crate::planner_entries::models::PlanOutputInputs; use crate::planner_resolution::main::references::resolve_function_sql::resolve_function_sql; use crate::planner_strategy::main::audit_entries::build_selected_audit_entries::build_selected_audit_entries; use crate::planner_strategy::main::model_entries::extract_seed_columns::extract_seed_columns; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::record_class::record_class; @@ -39,7 +36,6 @@ use sqlbuild_python_project::resource_views::main::function_reads::source_file_p use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; use sqlbuild_python_project::sql_test_glue::main::assemble_sql_test_plans::assemble_sql_test_plans; -use sqlbuild_python_project::sql_test_glue::main::sql_test_model_chain_names_by_key_function::sql_test_model_chain_names_by_key_function; use sqlbuild_python_project::sql_test_glue::models::SqlTestAssembly; const LOADER_CONSUMER: &str = "loader"; @@ -775,7 +771,7 @@ pub(crate) fn plan_selected_sql_tests<'py>( /// Selected SQL test entries and warnings, optionally narrowed to one parameterized case. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn build_selected_test_entries<'py>( +pub(crate) fn build_selected_test_entries<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult<(Bound<'py, PyList>, Bound<'py, PyList>)> { @@ -799,8 +795,3 @@ fn build_selected_test_entries<'py>( PyList::new(py, selected.warnings)?, )) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(sql_test_model_chain_names_by_key_function(module)?)?; - module.add_function(wrap_pyfunction!(build_selected_test_entries, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs index ec3c28ce35..1fbe363f6f 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs @@ -6,16 +6,13 @@ use std::sync::{Mutex, PoisonError}; use std::time::{Duration, Instant}; use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::PyDict; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::{pyclass, pyfunction, pymethods}; use crate::planner_entries::_helpers::plans::plan_output::plan_selected_sql_tests; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_threads::main::signal_interpreter_exit::signal_interpreter_exit; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -215,7 +212,7 @@ pub(crate) fn start_planning( /// Join SQL test planning threads at exit: finalization ending one inside native planning aborts. #[pyfunction] -fn join_sql_test_planning_threads(py: Python<'_>) -> PyResult<()> { +pub(crate) fn join_sql_test_planning_threads(py: Python<'_>) -> PyResult<()> { signal_interpreter_exit(); let threads: &PlanningThreads = planning_threads(py)?.get(); py.detach(|| threads.wait(PLANNING_EXIT_JOIN)); @@ -225,7 +222,7 @@ fn join_sql_test_planning_threads(py: Python<'_>) -> PyResult<()> { /// Start SQL test planning on a native thread, or resolve to no tests when disabled. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn start_sql_test_planning<'py>( +pub(crate) fn start_sql_test_planning<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -255,13 +252,3 @@ fn start_sql_test_planning<'py>( args.flag("enabled")?, ) } - -/// Add the planning handle and entry points, and join planning threads at interpreter exit. -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(start_sql_test_planning, module)?)?; - let hook = wrap_pyfunction!(join_sql_test_planning_threads, module)?; - module.add_function(hook.clone())?; - imported(module.py(), "atexit", "register")?.call1((hook,))?; - Ok(()) -} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/default_microbatch_limit_action.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/default_microbatch_limit_action.rs index ec25bb28d4..931c88d0f6 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/default_microbatch_limit_action.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/default_microbatch_limit_action.rs @@ -1,13 +1,10 @@ //! The microbatch limit action used when a project configures none. -use crate::planner_entries::_helpers::entries::contexts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::types::RecordId; /// The microbatch limit action used when a project configures none. -#[inline] pub fn default_microbatch_limit_action(py: Python<'_>) -> PyResult> { - contexts::default_microbatch_limit_action(py) + enum_member(py, RecordId::MicrobatchLimitAction, "ERROR") } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_model.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_model.rs index 586526494e..de7eebbf54 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_model.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_model.rs @@ -1,17 +1,75 @@ //! Detect changes and build a plan entry and warnings for a single model. -use crate::planner_entries::_helpers::entries::model_entries; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::planner_changes::main::detect_model_changes::detect_model_changes; +use crate::planner_entries::_helpers::entries::model_entries::ContextReads; +use crate::planner_entries::_helpers::entries::model_entries::ModelPlanRequest; +use crate::planner_entries::_helpers::entries::model_entries::plan_model_from_change; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Detect changes and build a plan entry and warnings for a single model. -#[inline] +#[pyfunction] +#[pyo3(signature = (**kwargs))] pub fn plan_model<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - model_entries::plan_model(py, kwargs) + let args = KeywordArguments::read( + py, + "plan_model", + kwargs, + ( + &[ + "model", + "snapshot", + "adapter", + "context", + "sql_analysis_enabled", + "query_change_tracking", + "full_refresh", + "cursor_overrides", + ], + &["external_sql_reference_resolver"], + ), + )?; + let model = args.value("model")?; + let snapshot = args.value("snapshot")?; + let change_result = detect_model_changes( + py, + Some(keywords( + py, + &[ + ("model", model.clone()), + ("snapshot", snapshot.clone()), + ("sql_analysis_enabled", args.value("sql_analysis_enabled")?), + ( + "query_change_tracking", + args.value("query_change_tracking")?, + ), + ("full_refresh", args.value("full_refresh")?), + ], + )?), + )?; + let overrides = args.value("cursor_overrides")?; + plan_model_from_change(&ModelPlanRequest { + model, + snapshot, + context: &ContextReads::of( + args.value("context")?, + args.value("adapter")?, + args.value("external_sql_reference_resolver")?, + )?, + sql_analysis_enabled: args.flag("sql_analysis_enabled")?, + full_refresh: args.flag("full_refresh")?, + overrides: ( + native_field(&overrides, "start_cursor_override")?, + native_field(&overrides, "end_cursor_override")?, + ), + change_result, + protect_full_refresh: true, + }) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/planner_relations_context.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/planner_relations_context.rs index cb95c5607c..cb9f74ca22 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/planner_relations_context.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/planner_relations_context.rs @@ -2,9 +2,7 @@ use crate::planner_entries::_helpers::entries::relations; use crate::planner_entries::models::RelationsRequest; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Resolve relation locations and source metadata for plan entry construction. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_model_entries.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_model_entries.rs index a72331b11a..7a6254ff28 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_model_entries.rs @@ -1,12 +1,12 @@ //! Register the model entries functions and classes. -use crate::planner_entries::_helpers::entries::model_entries; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_entries::_helpers::entries::model_entries::scope_overlaps; +use crate::planner_entries::main::entries::plan_model::plan_model; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the model entries functions and classes. -#[inline] pub fn register_model_entries(module: &Bound<'_, PyModule>) -> PyResult<()> { - model_entries::register(module) + module.add_function(wrap_pyfunction!(plan_model, module)?)?; + module.add_function(wrap_pyfunction!(scope_overlaps, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_relations.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_relations.rs index 8c37b936f1..741b300888 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_relations.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_relations.rs @@ -1,12 +1,10 @@ //! Register the relations functions and classes. -use crate::planner_entries::_helpers::entries::relations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_entries::_helpers::entries::relations::build_planner_relations_context; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the relations functions and classes. -#[inline] pub fn register_relations(module: &Bound<'_, PyModule>) -> PyResult<()> { - relations::register(module) + module.add_function(wrap_pyfunction!(build_planner_relations_context, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs index 9f640a4d3c..9ae83b6857 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs @@ -1,17 +1,59 @@ //! Project selected audits from canonical static state. -use crate::planner_entries::_helpers::plans::command_plans; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_entries::_helpers::plans::command_plans::StaticPlanParts; +use crate::planner_entries::_helpers::plans::command_plans::base_plan_output; +use crate::planner_strategy::main::audit_entries::build_selected_audit_entries::build_selected_audit_entries; +use crate::planner_strategy::main::strategy::materialization_of::materialization_of; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::planner_phases::main::object::object; /// Project selected audits from canonical static state. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project, adapter, scope, relations))] pub fn build_audit_command_plan<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, scope: &Bound<'py, PyAny>, relations: &Bound<'py, PyAny>, ) -> PyResult> { - command_plans::build_audit_command_plan(project, adapter, scope, relations) + let py = project.py(); + let materializations = PyDict::new(py); + for model in compiled_models(project)?.try_iter()? { + let model = model?; + materializations.set_item( + name_object(&model)?, + native_field(&materialization_of(&model)?, "value")?, + )?; + } + let audits = object( + py, + build_selected_audit_entries(Some(&keywords( + py, + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("scope", scope.clone()), + ("relations", relations.clone()), + ("model_materializations", materializations.into_any()), + ], + )?))?, + )?; + base_plan_output( + project, + scope, + relations, + StaticPlanParts { + execution_order: None, + fields: vec![( + "audit_entries", + PyTuple::new(py, audits.try_iter()?.collect::>>()?)?.into_any(), + )], + }, + ) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_display_only_sqlbuild_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_display_only_sqlbuild_plan.rs index 799a541dfc..b030728547 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_display_only_sqlbuild_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_display_only_sqlbuild_plan.rs @@ -1,16 +1,58 @@ //! Build a display-only plan when warehouse planning cannot fully resolve. -use crate::planner_entries::_helpers::plans::display_plan; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_entries::_helpers::entries::values::planner_model; +use crate::planner_entries::_helpers::plans::display_plan::display_entry; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyFrozenSet, PyTuple}; +use sqlbuild_analysis::graph::main::downstream_edges::downstream_edges; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::graph::main::project::edge_dict::edge_dict; +use sqlbuild_python_project::graph::models::KeyCache; +use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; /// Build a display-only plan when warehouse planning cannot fully resolve. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project, selected_model_names, full_refresh))] pub fn build_display_only_sqlbuild_plan<'py>( project: &Bound<'py, PyAny>, selected_model_names: &Bound<'py, PyAny>, full_refresh: bool, ) -> PyResult> { - display_plan::build_display_only_sqlbuild_plan(project, selected_model_names, full_refresh) + let py: Python<'py> = project.py(); + let selected = PyFrozenSet::new( + py, + selected_model_names + .try_iter()? + .collect::>>()?, + )?; + let mut entries: Vec> = Vec::new(); + for model in compiled_models(project)?.try_iter()? { + let model = model?; + if selected.contains(name_object(&model)?)? { + entries.push(display_entry(&model, full_refresh)?); + } + } + let keys: Vec> = entries + .iter() + .map(|entry| native_field(entry, "key")) + .collect::>()?; + let upstream = execution_deps(project)?; + let cache = KeyCache::new(py)?; + let downstream = edge_dict(&cache, &downstream_edges(&upstream))?.into_any(); + let upstream = edge_dict(&cache, &upstream)?.into_any(); + planner_model( + py, + RecordId::PlanOutput, + &[ + ("execution_order", PyTuple::new(py, &keys)?.into_any()), + ("model_entries", PyTuple::new(py, entries)?.into_any()), + ("selected_keys", PyFrozenSet::new(py, keys)?.into_any()), + ("upstream_deps", upstream), + ("downstream_deps", downstream), + ], + ) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs index 97c354073e..a854ea1cda 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs @@ -1,17 +1,85 @@ //! Build the execution plan for one selection against live warehouse state. -use crate::planner_entries::_helpers::plans::execution_plan; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::planner_entries::_helpers::plans::execution_plan::INSPECTION_CATALOG; +use crate::planner_entries::_helpers::plans::execution_plan::planned_output; +use crate::planner_entries::_helpers::plans::execution_plan::scopes_and_cursor_overrides; +use crate::planner_entries::_helpers::plans::test_planning::{PlanningRequest, start_planning}; +use crate::planner_phases::main::inputs::class_field::class_field; +use crate::planner_phases::models::PlannerRuntime; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_command::main::utc_now::utc_now; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; /// Build the execution plan for one selection against live warehouse state. -#[inline] +#[pyfunction] +#[pyo3(signature = (**kwargs))] pub fn build_execution_plan<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { - execution_plan::build_execution_plan(py, kwargs) + let args = KeywordArguments::read( + py, + "build_execution_plan", + kwargs, + ( + &[ + "project", + "adapter", + "connection", + "selection", + "overrides", + "deferral", + "policies", + ], + &["on_progress", "project_config", "local_config"], + ), + )?; + let project = args.value("project")?; + let adapter = args.value("adapter")?; + let connection = args.value("connection")?; + let runtime = Bound::new( + py, + PlannerRuntime { + project: project.clone().unbind(), + adapter: adapter.clone().unbind(), + connection: connection.clone().unbind(), + project_config: args.value("project_config")?.unbind(), + local_config: args.value("local_config")?.unbind(), + on_progress: args.value("on_progress")?.unbind(), + invocation_time: utc_now(py)?.unbind(), + }, + )? + .into_any(); + let (scopes, overrides) = scopes_and_cursor_overrides(&args)?; + let inspection = class_field(&scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?; + let test_planning = start_planning( + py, + PlanningRequest { + project: project.clone().unbind(), + adapter: adapter.clone().unbind(), + selected_keys: native_field(&inspection, "selected_keys")?.unbind(), + sql_test_selection: native_field(&inspection, "sql_test_selection")?.unbind(), + }, + native_field(&args.value("policies")?, "plan_sql_tests")?.is_truthy()?, + )?; + let planned = (|| { + let catalog = call_with( + &host_entry(py, INSPECTION_CATALOG)?, + &[("adapter", adapter), ("connection", connection)], + )?; + within(&catalog, |_| { + planned_output(&args, &runtime, (&scopes, &overrides), test_planning.get()) + }) + })(); + let joined = test_planning.get().result(py); + let planned = planned?; + joined?; + Ok(planned) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_relation_command_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_relation_command_plan.rs index 67f9d38b94..fb0a192aed 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_relation_command_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_relation_command_plan.rs @@ -1,16 +1,14 @@ //! Project static relation data for Python checks. -use crate::planner_entries::_helpers::plans::command_plans; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_entries::_helpers::plans::command_plans::StaticPlanParts; +use crate::planner_entries::_helpers::plans::command_plans::base_plan_output; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Project static relation data for Python checks. -#[inline] pub fn build_relation_command_plan<'py>( project: &Bound<'py, PyAny>, scope: &Bound<'py, PyAny>, relations: &Bound<'py, PyAny>, ) -> PyResult> { - command_plans::build_relation_command_plan(project, scope, relations) + base_plan_output(project, scope, relations, StaticPlanParts::default()) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_seed_command_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_seed_command_plan.rs index 2620e8e657..5996a0b5bf 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_seed_command_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_seed_command_plan.rs @@ -1,9 +1,7 @@ //! Project selected direct seed work and canonical identities. use crate::planner_entries::_helpers::plans::command_plans; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Project selected direct seed work and canonical identities. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_test_command_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_test_command_plan.rs index 4f87b69a22..b5a78d1b67 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_test_command_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_test_command_plan.rs @@ -1,10 +1,7 @@ //! Project selected SQL tests from canonical static state. use crate::planner_entries::_helpers::plans::command_plans; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyDict; /// Project selected SQL tests from canonical static state. diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/plan_output_fields.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/plan_output_fields.rs index db4f0e1810..d6add1d6d6 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/plan_output_fields.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/plan_output_fields.rs @@ -2,9 +2,7 @@ use crate::planner_entries::_helpers::plans::plan_output; use crate::planner_entries::models::PlanOutputInputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// The plan output's fields from planned entries and resolved relation inputs. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/read_selected_seed_fingerprints.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/read_selected_seed_fingerprints.rs index e98225ca3a..2a833b1c09 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/read_selected_seed_fingerprints.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/read_selected_seed_fingerprints.rs @@ -1,9 +1,7 @@ //! Read only fingerprint state required by selected direct seeds. use crate::planner_entries::_helpers::plans::command_plans; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; /// Read only fingerprint state required by selected direct seeds. diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_clone_plans.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_clone_plans.rs index 9f4b261d29..3f827aacd0 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_clone_plans.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_clone_plans.rs @@ -1,12 +1,12 @@ //! Register the clone plans functions and classes. -use crate::planner_entries::_helpers::plans::clone_plans; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_entries::_helpers::plans::clone_plans::resolve_clone_boundary; +use crate::planner_entries::_helpers::plans::clone_plans::resolve_skipped_view_chain; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the clone plans functions and classes. -#[inline] pub fn register_clone_plans(module: &Bound<'_, PyModule>) -> PyResult<()> { - clone_plans::register(module) + module.add_function(wrap_pyfunction!(resolve_clone_boundary, module)?)?; + module.add_function(wrap_pyfunction!(resolve_skipped_view_chain, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_command_plans.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_command_plans.rs index 96f1bfd2f5..3d5bd5f991 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_command_plans.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_command_plans.rs @@ -1,12 +1,20 @@ //! Register the command plans functions and classes. -use crate::planner_entries::_helpers::plans::command_plans; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_entries::_helpers::plans::command_plans::build_seed_command_plan; +use crate::planner_entries::_helpers::plans::command_plans::build_test_command_plan; +use crate::planner_entries::_helpers::plans::command_plans::read_selected_seed_fingerprints; +use crate::planner_entries::main::plans::build_audit_command_plan::build_audit_command_plan; +use crate::planner_entries::main::plans::resolve_static_command_scope::resolve_static_command_scope; +use crate::planner_entries::main::plans::resolve_static_relation_context::resolve_static_relation_context; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the command plans functions and classes. -#[inline] pub fn register_command_plans(module: &Bound<'_, PyModule>) -> PyResult<()> { - command_plans::register(module) + module.add_function(wrap_pyfunction!(resolve_static_command_scope, module)?)?; + module.add_function(wrap_pyfunction!(resolve_static_relation_context, module)?)?; + module.add_function(wrap_pyfunction!(build_audit_command_plan, module)?)?; + module.add_function(wrap_pyfunction!(build_test_command_plan, module)?)?; + module.add_function(wrap_pyfunction!(build_seed_command_plan, module)?)?; + module.add_function(wrap_pyfunction!(read_selected_seed_fingerprints, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_display_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_display_plan.rs index c69fa8c119..a248a2ee36 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_display_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_display_plan.rs @@ -1,12 +1,10 @@ //! Register the display plan functions and classes. -use crate::planner_entries::_helpers::plans::display_plan; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_entries::main::plans::build_display_only_sqlbuild_plan::build_display_only_sqlbuild_plan; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the display plan functions and classes. -#[inline] pub fn register_display_plan(module: &Bound<'_, PyModule>) -> PyResult<()> { - display_plan::register(module) + module.add_function(wrap_pyfunction!(build_display_only_sqlbuild_plan, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_execution_plan.rs index 24db27cb5f..1572e85575 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_execution_plan.rs @@ -1,12 +1,10 @@ //! Add the execution plan entry point. -use crate::planner_entries::_helpers::plans::execution_plan; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_entries::main::plans::build_execution_plan::build_execution_plan; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Add the execution plan entry point. -#[inline] pub fn register_execution_plan(module: &Bound<'_, PyModule>) -> PyResult<()> { - execution_plan::register(module) + module.add_function(wrap_pyfunction!(build_execution_plan, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_plan_output.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_plan_output.rs index f589b81332..087d96c6f9 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_plan_output.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_plan_output.rs @@ -1,12 +1,12 @@ //! Register the plan output functions and classes. -use crate::planner_entries::_helpers::plans::plan_output; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_entries::_helpers::plans::plan_output::build_selected_test_entries; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; +use sqlbuild_python_project::sql_test_glue::main::sql_test_model_chain_names_by_key_function::sql_test_model_chain_names_by_key_function; /// Register the plan output functions and classes. -#[inline] pub fn register_plan_output(module: &Bound<'_, PyModule>) -> PyResult<()> { - plan_output::register(module) + module.add_function(sql_test_model_chain_names_by_key_function(module)?)?; + module.add_function(wrap_pyfunction!(build_selected_test_entries, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_test_planning.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_test_planning.rs index 4275aa0909..be6dc52d85 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_test_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_test_planning.rs @@ -1,12 +1,18 @@ //! Add the planning handle and entry points, and join planning threads at interpreter exit. -use crate::planner_entries::_helpers::plans::test_planning; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_entries::_helpers::plans::test_planning::SqlTestPlanning; +use crate::planner_entries::_helpers::plans::test_planning::join_sql_test_planning_threads; +use crate::planner_entries::_helpers::plans::test_planning::start_sql_test_planning; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; +use sqlbuild_python_base::boundary::main::imported::imported; /// Add the planning handle and entry points, and join planning threads at interpreter exit. -#[inline] pub fn register_test_planning(module: &Bound<'_, PyModule>) -> PyResult<()> { - test_planning::register(module) + module.add_class::()?; + module.add_function(wrap_pyfunction!(start_sql_test_planning, module)?)?; + let hook = wrap_pyfunction!(join_sql_test_planning_threads, module)?; + module.add_function(hook.clone())?; + imported(module.py(), "atexit", "register")?.call1((hook,))?; + Ok(()) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_command_scope.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_command_scope.rs index 9f2dd34476..e9d0b5592e 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_command_scope.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_command_scope.rs @@ -1,16 +1,28 @@ //! Resolve command selection through the canonical build selector phase. -use crate::planner_entries::_helpers::plans::command_plans; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_entries::_helpers::entries::values::{boolean, planner_model}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_scope::main::resolve_planner_scopes::resolve_planner_scopes; /// Resolve command selection through the canonical build selector phase. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project, selection, auto_load_sources = false))] pub fn resolve_static_command_scope<'py>( project: &Bound<'py, PyAny>, selection: &Bound<'py, PyAny>, auto_load_sources: bool, ) -> PyResult> { - command_plans::resolve_static_command_scope(project, selection, auto_load_sources) + let py = project.py(); + resolve_planner_scopes( + project, + selection, + &planner_model( + py, + RecordId::PlannerPolicies, + &[("auto_load_sources", boolean(py, auto_load_sources)?)], + )?, + )? + .getattr("selected_scope") } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs index f15397d452..63cc2accfc 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs @@ -1,17 +1,43 @@ //! Resolve canonical locations and source routing without warehouse inventory. -use crate::planner_entries::_helpers::plans::command_plans; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::planner_entries::_helpers::entries::relations::planner_relations_context; +use crate::planner_entries::_helpers::entries::values::none; +use crate::planner_entries::models::RelationsRequest; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; /// Resolve canonical locations and source routing without warehouse inventory. -#[inline] +#[pyfunction] +#[pyo3(signature = (**kwargs))] pub fn resolve_static_relation_context<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { - command_plans::resolve_static_relation_context(py, kwargs) + let args = KeywordArguments::read( + py, + "resolve_static_relation_context", + kwargs, + ( + &["project", "adapter", "scope"], + &[ + "deferral", + "project_config", + "local_config", + "relation_keys", + ], + ), + )?; + planner_relations_context(RelationsRequest { + project: args.value("project")?, + adapter: args.value("adapter")?, + connection: none(py), + scope: args.value("scope")?, + deferral: args.value("deferral")?, + project_config: args.value("project_config")?, + local_config: args.value("local_config")?, + known_source_columns: PyDict::new(py).into_any(), + relation_keys: args.value("relation_keys")?, + }) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/timed_clone_planning.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/timed_clone_planning.rs index 1e7f88937d..d4f4bc552b 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/timed_clone_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/timed_clone_planning.rs @@ -2,11 +2,8 @@ use crate::planner_entries::_helpers::plans::clone_plans; use crate::planner_entries::types::PhaseTimings; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; /// Plan a clone, with the seconds spent inspecting sources and planning. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_entries/models.rs b/crates/sqlbuild-python-planner/src/planner_entries/models.rs index cfd87635c9..91867bcf9b 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/models.rs @@ -1,8 +1,6 @@ //! Public records of planner entries. -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; +use pyo3::prelude::{Bound, Py, PyAny}; use pyo3::pyclass; use sqlbuild_python_project::native_host::models::ModelLookup; diff --git a/crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs index 626150f3be..68240b5a93 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs @@ -1,10 +1,12 @@ //! Direct model version identities and honest write hashes over the planner scope. +use crate::planner_identity::main::hook_version_hashes::hook_version_hashes; +use crate::planner_identity::main::model_version_identity_metadata_json::model_version_identity_metadata_json; use std::collections::{HashMap, HashSet}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_identity::constants::{ MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE, TABLE_FN_RESOURCE, UDF_RESOURCE, @@ -13,13 +15,9 @@ use sqlbuild_analysis::planner_identity::main::expected_graph_identity_hashes::e use sqlbuild_analysis::planner_identity::main::graph_write_identity_hashes::graph_write_identity_hashes; use sqlbuild_analysis::planner_identity::models::{GraphKey, IdentityGraph, IdentityNode}; use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; -use sqlbuild_host::entries::models::UserCodeHost; -use crate::planner_identity::_helpers::metadata::function_local_hashes; -use crate::planner_identity::_helpers::metadata::model_version_identity_metadata_json; -use crate::planner_identity::_helpers::metadata::seed_identity; +use crate::planner_identity::_helpers::metadata::{function_local_hashes, seed_identity}; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; @@ -51,21 +49,6 @@ fn string_pairs(mapping: &Bound<'_, PyAny>) -> PyResult> { .collect() } -/// `{name: identity.version_hash}` of the discovered Python hooks. -pub(crate) fn hook_version_hashes<'py>( - hook_functions: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = hook_functions.py(); - let hashes: Bound<'py, PyDict> = PyDict::new(py); - let identities: Bound<'py, PyAny> = - host_entry(py, UserCodeHost::BuildHookIdentities)?.call1((hook_functions,))?; - for item in mapping_items(&identities)?.try_iter()? { - let (name, identity): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; - hashes.set_item(name, native_field(&identity, "version_hash")?)?; - } - Ok(hashes) -} - /// Current direct model identities from code and upstream identities. #[pyfunction] #[pyo3(signature = (**options))] @@ -382,41 +365,9 @@ pub(crate) fn with_honest_model_write_hashes<'py>( replace_with(&changes, ("models", model_changes.into_any())) } -/// Resolved actions with recomputed model changes patched in where available. -#[pyfunction] -#[pyo3(signature = (*, resolved_actions, changes))] -pub(crate) fn merge_recomputed_model_changes<'py>( - resolved_actions: &Bound<'py, PyAny>, - changes: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = resolved_actions.py(); - let recomputed: Bound<'py, PyAny> = native_field(changes, "models")?; - let merged: Bound<'py, PyDict> = PyDict::new(py); - for item in native_field(resolved_actions, "models")? - .call_method0("items")? - .try_iter()? - { - let (name, resolved): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; - let mut change: Bound<'py, PyAny> = mapping_get(&recomputed, &name)?; - if change.is_none() { - change = native_field(&resolved, "change")?; - } - merged.set_item(name, replace_with(&resolved, ("change", change))?)?; - } - replace_with(resolved_actions, ("models", merged.into_any())) -} - -fn replace_with<'py>( +pub(crate) fn replace_with<'py>( instance: &Bound<'py, PyAny>, (field, value): (&str, Bound<'py, PyAny>), ) -> PyResult> { replaced(instance, &[(field, value)]) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - build_direct_model_version_identities, - module - )?)?; - module.add_function(wrap_pyfunction!(merge_recomputed_model_changes, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs index 235e4a8444..21bbcc1793 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs @@ -1,19 +1,19 @@ //! Version identity metadata JSON and hashes of models, functions and seeds, byte-identical to the recorded fingerprints. +use crate::planner_identity::main::compiled_function_fingerprint_sql::compiled_function_fingerprint_sql; +use crate::planner_identity::main::model_declared_columns_hash::model_declared_columns_hash; +use crate::planner_identity::main::or_empty::or_empty; use std::collections::HashMap; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_identity::constants::{ CONTRACT_CONFIG_KEY, CURSOR_INPUTS_CONFIG_KEY, ENFORCED_CONTRACT_POLICY, MODEL_CUSTOM_CONFIG_KEY, MODEL_PLACEHOLDERS_CONFIG_KEY, MODEL_POST_HOOKS_CONFIG_KEY, - MODEL_PRE_HOOKS_CONFIG_KEY, SQL_HOOK_IDENTITY_FIELDS, TABLE_FN_RESOURCE, UDF_RESOURCE, - VERSION_IDENTITY_CONFIG_KEYS, + MODEL_PRE_HOOKS_CONFIG_KEY, SQL_HOOK_IDENTITY_FIELDS, VERSION_IDENTITY_CONFIG_KEYS, }; -use sqlbuild_analysis::planner_identity::main::function_fingerprint_sql::function_fingerprint_sql; -use sqlbuild_analysis::planner_identity::models::FunctionFingerprintFields; use sqlbuild_core::fingerprint_identity::main::stable_identity_hash::stable_identity_hash; use crate::microbatches::main::timestamp::timestamp; @@ -27,11 +27,6 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_front::discovery::main::headers::hook_entries::hook_entries; -use sqlbuild_python_front::model_config::main::config::config_value::config_value; -use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::compiled_models::config::config; -use sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps; -use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::planner_identity::main::baselined_function_definition_hash::baselined_function_definition_hash; use sqlbuild_python_project::planner_identity::main::function_definition_hash::function_definition_hash; use sqlbuild_python_project::resource_views::main::function_reads::arguments::arguments; @@ -69,17 +64,6 @@ fn version_identity_config<'py>(config_values: &Bound<'py, PyAny>) -> PyResult( - value: Option>, - py: Python<'py>, -) -> PyResult> { - match value { - Some(value) if value.is_truthy()? => Ok(value), - _ => Ok(PyDict::new(py).into_any()), - } -} - /// Deterministic metadata JSON for model version identity. pub(crate) fn version_identity_metadata_json( model_name: &Bound<'_, PyAny>, @@ -101,44 +85,7 @@ pub(crate) fn version_identity_metadata_json( sorted_dumps(payload.as_any(), COMPACT, true) } -/// Deterministic non-query model fingerprint metadata JSON. -pub(crate) fn model_version_identity_metadata_json<'py>( - model: &Bound<'py, PyAny>, - function_local_hashes: Option<&Bound<'py, PyAny>>, - hook_version_hashes: Option<&Bound<'py, PyAny>>, -) -> PyResult { - let py: Python<'py> = model.py(); - let local_function_hashes: Bound<'py, PyDict> = PyDict::new(py); - if let Some(function_hashes) = function_local_hashes.filter(|hashes| !hashes.is_none()) { - for upstream_key in deps(model)?.try_iter()? { - let upstream_key: Bound<'py, PyAny> = upstream_key?; - let kind: String = native_field(&upstream_key, "resource_type")? - .str()? - .extract()?; - if kind != UDF_RESOURCE && kind != TABLE_FN_RESOURCE { - continue; - } - let name: Bound<'py, PyAny> = native_field(&upstream_key, "name")?; - let upstream_hash: Bound<'py, PyAny> = mapping_get(function_hashes, &name)?; - if !upstream_hash.is_none() { - local_function_hashes.set_item(name, upstream_hash)?; - } - } - } - let config_values: Bound<'py, PyAny> = native_field(&config(model)?, "values")?; - let signature: Bound<'py, PyDict> = - execution_signature(model, &config_values, hook_version_hashes)?; - version_identity_metadata_json( - &name_object(model)?, - &config_values, - ( - Some(local_function_hashes.into_any()), - Some(signature.into_any()), - ), - ) -} - -fn execution_signature<'py>( +pub(crate) fn execution_signature<'py>( model: &Bound<'py, PyAny>, config_values: &Bound<'py, PyAny>, hook_version_hashes: Option<&Bound<'py, PyAny>>, @@ -242,7 +189,7 @@ fn contract_output_signature<'py>( Ok(Some(signature)) } -fn column_signature<'py>(column: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn column_signature<'py>(column: &Bound<'py, PyAny>) -> PyResult> { let entry: Bound<'py, PyDict> = PyDict::new(column.py()); for field in ["name", "type", "nullable"] { entry.set_item(field, native_field(column, field)?)?; @@ -250,7 +197,7 @@ fn column_signature<'py>(column: &Bound<'py, PyAny>) -> PyResult(families: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn family_signatures<'py>(families: &Bound<'py, PyAny>) -> PyResult> { let rendered: Bound<'py, PyList> = PyList::empty(families.py()); for family in families.try_iter()? { let family: Bound<'py, PyAny> = family?; @@ -270,38 +217,6 @@ fn family_signatures<'py>(families: &Bound<'py, PyAny>) -> PyResult) -> PyResult { - let py: Python<'_> = model.py(); - let schema_entry: Bound<'_, PyAny> = - sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( - model, - )?; - let payload: Bound<'_, PyDict> = PyDict::new(py); - payload.set_item( - "contract", - config_value(py, &model_values(model)?, CONTRACT_CONFIG_KEY)?, - )?; - let columns: Bound<'_, PyList> = PyList::empty(py); - let mut families: Bound<'_, PyList> = PyList::empty(py); - let mut type_enforcement: Bound<'_, PyAny> = py.None().into_bound(py); - if !schema_entry.is_none() { - for column in native_field(&schema_entry, "columns")?.try_iter()? { - columns.append(column_signature(&column?)?)?; - } - families = family_signatures(&native_field(&schema_entry, "dynamic_columns")?)?; - type_enforcement = native_field(&schema_entry, "type_enforcement")?; - } - payload.set_item("columns", columns)?; - payload.set_item("dynamic_columns", families)?; - payload.set_item("type_enforcement", type_enforcement)?; - Ok(stable_identity_hash(&sorted_dumps( - payload.as_any(), - COMPACT, - true, - )?)) -} - /// Whether the function's recorded fingerprint came from an older scheme for the same definition. fn recorded_function_unchanged( function: &Bound<'_, PyAny>, @@ -438,7 +353,7 @@ fn text_or_empty(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { } /// `f"{value or ''}"` of an attribute read, empty when the read failed. -fn text_of(read: PyResult>) -> PyResult { +pub(crate) fn text_of(read: PyResult>) -> PyResult { let value: Bound<'_, PyAny> = match read { Ok(value) => value, Err(_) => return Ok(String::new()), @@ -450,7 +365,7 @@ fn text_of(read: PyResult>) -> PyResult { } } -fn rendered_pairs(items: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn rendered_pairs(items: &Bound<'_, PyAny>) -> PyResult> { items .try_iter()? .map(|item| { @@ -460,28 +375,6 @@ fn rendered_pairs(items: &Bound<'_, PyAny>) -> PyResult> { .collect() } -/// Stable function definition text used for change fingerprints. -pub(crate) fn compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { - Ok(function_fingerprint_sql(&FunctionFingerprintFields { - language: - sqlbuild_python_project::resource_views::main::function_reads::language::language( - function, - )? - .str()? - .extract()?, - arguments: rendered_pairs(&arguments(function)?)?, - returns: returns(function)?.str()?.extract()?, - return_columns: rendered_pairs(&return_columns(function)?)?, - runtime_version: text_of(runtime_version(function))?, - entry_point: text_of(entry_point(function))?, - packages: packages(function)? - .try_iter()? - .map(|package| package?.extract::()) - .collect::>()?, - body_sql: body_sql(function)?.extract()?, - })) -} - /// `(identity_hash, metadata_json)` of one compiled seed from its content and load-affecting config. pub(crate) fn seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { let py: Python<'_> = seed.py(); @@ -561,7 +454,7 @@ fn seed_rows<'py>( /// Deterministic metadata JSON for model version identity. #[pyfunction] #[pyo3(signature = (*, model_name, config_values, local_function_hashes = None, execution_signature = None))] -fn build_version_identity_metadata_json<'py>( +pub(crate) fn build_version_identity_metadata_json<'py>( model_name: &Bound<'py, PyAny>, config_values: &Bound<'py, PyAny>, local_function_hashes: Option>, @@ -574,70 +467,9 @@ fn build_version_identity_metadata_json<'py>( ) } -/// Deterministic non-query model fingerprint metadata JSON. -#[pyfunction] -#[pyo3(signature = (*, model, function_local_hashes = None, hook_version_hashes = None))] -pub(crate) fn build_model_version_identity_metadata_json<'py>( - model: &Bound<'py, PyAny>, - function_local_hashes: Option>, - hook_version_hashes: Option>, -) -> PyResult { - model_version_identity_metadata_json( - model, - function_local_hashes.as_ref(), - hook_version_hashes.as_ref(), - ) -} - /// Hash the model's own declared columns, column families and contract setting. #[pyfunction] #[pyo3(signature = (*, model))] -fn declared_columns_hash(model: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn declared_columns_hash(model: &Bound<'_, PyAny>) -> PyResult { model_declared_columns_hash(model) } - -/// Local-only semantic hashes of functions by name. -#[pyfunction] -#[pyo3(signature = (*, functions, dialect))] -pub(crate) fn build_function_local_hashes<'py>( - functions: &Bound<'py, PyAny>, - dialect: Option<&str>, -) -> PyResult> { - function_local_hashes( - functions, - dialect, - &functions.py().None().into_bound(functions.py()), - ) -} - -/// Stable function definition text used for change fingerprints. -#[pyfunction] -pub(crate) fn build_compiled_function_fingerprint_sql( - function: &Bound<'_, PyAny>, -) -> PyResult { - compiled_function_fingerprint_sql(function) -} - -/// `(identity_hash, metadata_json)` for one compiled seed. -#[pyfunction] -pub(crate) fn build_seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { - seed_identity(seed) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - build_version_identity_metadata_json, - module - )?)?; - module.add_function(wrap_pyfunction!( - build_model_version_identity_metadata_json, - module - )?)?; - module.add_function(wrap_pyfunction!(declared_columns_hash, module)?)?; - module.add_function(wrap_pyfunction!(build_function_local_hashes, module)?)?; - module.add_function(wrap_pyfunction!( - build_compiled_function_fingerprint_sql, - module - )?)?; - module.add_function(wrap_pyfunction!(build_seed_identity, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs index 69e575e0d5..1dc365c694 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs @@ -4,13 +4,10 @@ use crate::planner_changes::main::native_enum::native_enum; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use std::collections::{HashMap, HashSet}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyString, PyTuple}; -use sqlbuild_analysis::planner_identity::constants::{ - MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE, -}; use sqlbuild_analysis::planner_identity::main::classify_selection_staleness::classify_selection_staleness; use sqlbuild_analysis::planner_identity::main::stale_upstream_warning_message::stale_upstream_warning_message; use sqlbuild_analysis::planner_identity::models::{ @@ -19,7 +16,6 @@ use sqlbuild_analysis::planner_identity::models::{ use crate::planner_changes::main::model_change::model_change; use crate::planner_changes::models::ModelPass; -use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; @@ -55,15 +51,15 @@ fn upstream_graph(upstream_deps: &Bound<'_, PyAny>) -> PyResult { - original_scope: Bound<'py, PyAny>, - changes: Bound<'py, PyAny>, - snapshot: Bound<'py, PyAny>, - identities: Bound<'py, PyAny>, - model_changes_complete: bool, +pub(crate) struct StalenessInputs<'py> { + pub(crate) original_scope: Bound<'py, PyAny>, + pub(crate) changes: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) identities: Bound<'py, PyAny>, + pub(crate) model_changes_complete: bool, } -fn changed_model_names(inputs: &StalenessInputs<'_>) -> PyResult> { +pub(crate) fn changed_model_names(inputs: &StalenessInputs<'_>) -> PyResult> { let py: Python<'_> = inputs.snapshot.py(); let models: Bound<'_, PyAny> = native_field(&inputs.changes, "models")?; let mut changed: HashSet = HashSet::new(); @@ -114,7 +110,7 @@ fn changed_model_names(inputs: &StalenessInputs<'_>) -> PyResult Ok(changed) } -fn changed_seed_names(inputs: &StalenessInputs<'_>) -> PyResult> { +pub(crate) fn changed_seed_names(inputs: &StalenessInputs<'_>) -> PyResult> { let recorded: Bound<'_, PyAny> = native_field(&native_field(&inputs.snapshot, "fingerprints")?, "seeds")?; let mut changed: HashSet = HashSet::new(); @@ -134,7 +130,7 @@ fn changed_seed_names(inputs: &StalenessInputs<'_>) -> PyResult> Ok(changed) } -fn plan_warnings<'py>( +pub(crate) fn plan_warnings<'py>( py: Python<'py>, warnings: Vec, ) -> PyResult> { @@ -162,92 +158,13 @@ fn plan_warnings<'py>( PyTuple::new(py, rendered) } -/// Warn for selected models stale through changed upstreams outside the run set. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn build_stale_out_of_selection_warnings<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "build_stale_out_of_selection_warnings", - options, - ( - &[ - "original_scope", - "execution_scope", - "changes", - "snapshot", - "version_identities", - "source_freshness", - ], - &["include_sources", "model_changes_complete"], - ), - )?; - let include_sources: bool = - !arguments.passed("include_sources")? || arguments.flag("include_sources")?; - let inputs = StalenessInputs { - original_scope: arguments.value("original_scope")?, - changes: arguments.value("changes")?, - snapshot: arguments.value("snapshot")?, - identities: arguments.value("version_identities")?, - model_changes_complete: arguments.flag("model_changes_complete")?, - }; - let run = scope_data(&arguments.value("execution_scope")?)?; - let original = scope_data(&inputs.original_scope)?; - let names = |kind: &str| -> HashSet { - run.selected - .iter() - .filter(|key| key.0 == kind) - .map(|key| key.1.clone()) - .collect() - }; - let source_freshness: Bound<'py, PyAny> = arguments.value("source_freshness")?; - let mut changed_sources: HashSet = HashSet::new(); - if include_sources && !source_freshness.is_none() { - for identity in native_field(&source_freshness, "changed_identities")?.try_iter()? { - changed_sources.insert(native_field(&identity?, "source_name")?.extract()?); - } - } - let graph = StalenessGraph { - upstream_deps: original - .upstream - .iter() - .map(|(key, deps)| { - ( - GraphKey::from(key), - deps.iter().map(GraphKey::from).collect(), - ) - }) - .collect(), - selected_model_names: original - .selected - .iter() - .filter(|key| key.0 == MODEL_RESOURCE) - .map(|key| key.1.clone()) - .collect(), - run: StalenessNames { - models: names(MODEL_RESOURCE), - seeds: names(SEED_RESOURCE), - sources: names(SOURCE_RESOURCE), - }, - changed: StalenessNames { - models: changed_model_names(&inputs)?, - seeds: changed_seed_names(&inputs)?, - sources: changed_sources, - }, - }; - plan_warnings(py, classify_selection_staleness(&graph)) -} - fn name_set(values: &Bound<'_, PyAny>) -> PyResult> { values.try_iter()?.map(|value| value?.extract()).collect() } /// Stale warnings for selected models with changed upstreams outside the run set. #[pyfunction] -fn classify_selection_staleness_warnings<'py>( +pub(crate) fn classify_selection_staleness_warnings<'py>( graph: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = graph.py(); @@ -286,14 +203,3 @@ fn classify_selection_staleness_warnings<'py>( } PyTuple::new(py, warnings) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - build_stale_out_of_selection_warnings, - module - )?)?; - module.add_function(wrap_pyfunction!( - classify_selection_staleness_warnings, - module - )?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/build_compiled_function_fingerprint_sql.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/build_compiled_function_fingerprint_sql.rs index c18cbfb4f0..2cb92f048e 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/build_compiled_function_fingerprint_sql.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/build_compiled_function_fingerprint_sql.rs @@ -1,12 +1,11 @@ //! Stable function definition text used for change fingerprints. -use crate::planner_identity::_helpers::metadata; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_identity::main::compiled_function_fingerprint_sql::compiled_function_fingerprint_sql; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// Stable function definition text used for change fingerprints. -#[inline] +#[pyfunction] pub fn build_compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { - metadata::build_compiled_function_fingerprint_sql(function) + compiled_function_fingerprint_sql(function) } diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/build_function_local_hashes.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/build_function_local_hashes.rs index e44db01904..340537accd 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/build_function_local_hashes.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/build_function_local_hashes.rs @@ -1,16 +1,20 @@ //! Local-only semantic hashes of functions by name. -use crate::planner_identity::_helpers::metadata; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_identity::_helpers::metadata::function_local_hashes; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyDict; /// Local-only semantic hashes of functions by name. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, functions, dialect))] pub fn build_function_local_hashes<'py>( functions: &Bound<'py, PyAny>, dialect: Option<&str>, ) -> PyResult> { - metadata::build_function_local_hashes(functions, dialect) + function_local_hashes( + functions, + dialect, + &functions.py().None().into_bound(functions.py()), + ) } diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/build_model_version_identity_metadata_json.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/build_model_version_identity_metadata_json.rs index aa308c9928..8e3213981e 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/build_model_version_identity_metadata_json.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/build_model_version_identity_metadata_json.rs @@ -1,20 +1,20 @@ //! Deterministic non-query model fingerprint metadata JSON. -use crate::planner_identity::_helpers::metadata; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_identity::main::model_version_identity_metadata_json::model_version_identity_metadata_json; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// Deterministic non-query model fingerprint metadata JSON. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, model, function_local_hashes = None, hook_version_hashes = None))] pub fn build_model_version_identity_metadata_json<'py>( model: &Bound<'py, PyAny>, function_local_hashes: Option>, hook_version_hashes: Option>, ) -> PyResult { - metadata::build_model_version_identity_metadata_json( + model_version_identity_metadata_json( model, - function_local_hashes, - hook_version_hashes, + function_local_hashes.as_ref(), + hook_version_hashes.as_ref(), ) } diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/build_seed_identity.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/build_seed_identity.rs index 9269375d03..e0205428ad 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/build_seed_identity.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/build_seed_identity.rs @@ -1,12 +1,11 @@ //! `(identity_hash, metadata_json)` for one compiled seed. -use crate::planner_identity::_helpers::metadata; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_identity::_helpers::metadata::seed_identity; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// `(identity_hash, metadata_json)` for one compiled seed. -#[inline] +#[pyfunction] pub fn build_seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { - metadata::build_seed_identity(seed) + seed_identity(seed) } diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/build_stale_out_of_selection_warnings.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/build_stale_out_of_selection_warnings.rs index 71b9d7f830..67ea2a50db 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/build_stale_out_of_selection_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/build_stale_out_of_selection_warnings.rs @@ -1,17 +1,97 @@ //! Warn for selected models stale through changed upstreams outside the run set. -use crate::planner_identity::_helpers::staleness; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use crate::planner_identity::_helpers::staleness::StalenessInputs; +use crate::planner_identity::_helpers::staleness::changed_model_names; +use crate::planner_identity::_helpers::staleness::changed_seed_names; +use crate::planner_identity::_helpers::staleness::plan_warnings; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_analysis::planner_identity::constants::{ + MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE, +}; +use sqlbuild_analysis::planner_identity::main::classify_selection_staleness::classify_selection_staleness; +use sqlbuild_analysis::planner_identity::models::{GraphKey, StalenessGraph, StalenessNames}; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use std::collections::HashSet; /// Warn for selected models stale through changed upstreams outside the run set. -#[inline] +#[pyfunction] +#[pyo3(signature = (**options))] pub fn build_stale_out_of_selection_warnings<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - staleness::build_stale_out_of_selection_warnings(py, options) + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_stale_out_of_selection_warnings", + options, + ( + &[ + "original_scope", + "execution_scope", + "changes", + "snapshot", + "version_identities", + "source_freshness", + ], + &["include_sources", "model_changes_complete"], + ), + )?; + let include_sources: bool = + !arguments.passed("include_sources")? || arguments.flag("include_sources")?; + let inputs = StalenessInputs { + original_scope: arguments.value("original_scope")?, + changes: arguments.value("changes")?, + snapshot: arguments.value("snapshot")?, + identities: arguments.value("version_identities")?, + model_changes_complete: arguments.flag("model_changes_complete")?, + }; + let run = scope_data(&arguments.value("execution_scope")?)?; + let original = scope_data(&inputs.original_scope)?; + let names = |kind: &str| -> HashSet { + run.selected + .iter() + .filter(|key| key.0 == kind) + .map(|key| key.1.clone()) + .collect() + }; + let source_freshness: Bound<'py, PyAny> = arguments.value("source_freshness")?; + let mut changed_sources: HashSet = HashSet::new(); + if include_sources && !source_freshness.is_none() { + for identity in native_field(&source_freshness, "changed_identities")?.try_iter()? { + changed_sources.insert(native_field(&identity?, "source_name")?.extract()?); + } + } + let graph = StalenessGraph { + upstream_deps: original + .upstream + .iter() + .map(|(key, deps)| { + ( + GraphKey::from(key), + deps.iter().map(GraphKey::from).collect(), + ) + }) + .collect(), + selected_model_names: original + .selected + .iter() + .filter(|key| key.0 == MODEL_RESOURCE) + .map(|key| key.1.clone()) + .collect(), + run: StalenessNames { + models: names(MODEL_RESOURCE), + seeds: names(SEED_RESOURCE), + sources: names(SOURCE_RESOURCE), + }, + changed: StalenessNames { + models: changed_model_names(&inputs)?, + seeds: changed_seed_names(&inputs)?, + sources: changed_sources, + }, + }; + plan_warnings(py, classify_selection_staleness(&graph)) } diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/compiled_function_fingerprint_sql.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/compiled_function_fingerprint_sql.rs index 22d68f5197..14ddb5ab0f 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/compiled_function_fingerprint_sql.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/compiled_function_fingerprint_sql.rs @@ -1,12 +1,36 @@ //! Stable function definition text used for change fingerprints. -use crate::planner_identity::_helpers::metadata; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_identity::_helpers::metadata::rendered_pairs; +use crate::planner_identity::_helpers::metadata::text_of; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::planner_identity::main::function_fingerprint_sql::function_fingerprint_sql; +use sqlbuild_analysis::planner_identity::models::FunctionFingerprintFields; +use sqlbuild_python_project::resource_views::main::function_reads::arguments::arguments; +use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; +use sqlbuild_python_project::resource_views::main::function_reads::entry_point::entry_point; +use sqlbuild_python_project::resource_views::main::function_reads::packages::packages; +use sqlbuild_python_project::resource_views::main::function_reads::return_columns::return_columns; +use sqlbuild_python_project::resource_views::main::function_reads::returns::returns; +use sqlbuild_python_project::resource_views::main::function_reads::runtime_version::runtime_version; /// Stable function definition text used for change fingerprints. -#[inline] pub fn compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { - metadata::compiled_function_fingerprint_sql(function) + Ok(function_fingerprint_sql(&FunctionFingerprintFields { + language: + sqlbuild_python_project::resource_views::main::function_reads::language::language( + function, + )? + .str()? + .extract()?, + arguments: rendered_pairs(&arguments(function)?)?, + returns: returns(function)?.str()?.extract()?, + return_columns: rendered_pairs(&return_columns(function)?)?, + runtime_version: text_of(runtime_version(function))?, + entry_point: text_of(entry_point(function))?, + packages: packages(function)? + .try_iter()? + .map(|package| package?.extract::()) + .collect::>()?, + body_sql: body_sql(function)?.extract()?, + })) } diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/direct_model_version_identities.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/direct_model_version_identities.rs index e2df991bdf..2a4e65dd89 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/direct_model_version_identities.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/direct_model_version_identities.rs @@ -1,9 +1,7 @@ //! Direct model identities of `(functions, scope)`; unchanged nodes keep their `recorded` fingerprints. use crate::planner_identity::_helpers::identities; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Direct model identities of `(functions, scope)`; unchanged nodes keep their `recorded` fingerprints. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/function_local_hashes.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/function_local_hashes.rs index 04f88888ea..b4c9e4267c 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/function_local_hashes.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/function_local_hashes.rs @@ -1,9 +1,7 @@ //! Local hashes of functions by name; unchanged functions keep the latest local hash `recorded` models saw. use crate::planner_identity::_helpers::metadata; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; /// Local hashes of functions by name; unchanged functions keep the latest local hash `recorded` models saw. diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/hook_version_hashes.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/hook_version_hashes.rs index 49bb1068d4..d8ee24c528 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/hook_version_hashes.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/hook_version_hashes.rs @@ -1,15 +1,23 @@ //! `{name: identity.version_hash}` of the discovered Python hooks. -use crate::planner_identity::_helpers::identities; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; /// `{name: identity.version_hash}` of the discovered Python hooks. -#[inline] pub fn hook_version_hashes<'py>( hook_functions: &Bound<'py, PyAny>, ) -> PyResult> { - identities::hook_version_hashes(hook_functions) + let py: Python<'py> = hook_functions.py(); + let hashes: Bound<'py, PyDict> = PyDict::new(py); + let identities: Bound<'py, PyAny> = + host_entry(py, UserCodeHost::BuildHookIdentities)?.call1((hook_functions,))?; + for item in mapping_items(&identities)?.try_iter()? { + let (name, identity): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + hashes.set_item(name, native_field(&identity, "version_hash")?)?; + } + Ok(hashes) } diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/merge_recomputed_model_changes.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/merge_recomputed_model_changes.rs index e80012bf13..2cd53509c8 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/merge_recomputed_model_changes.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/merge_recomputed_model_changes.rs @@ -1,15 +1,32 @@ //! Resolved actions with recomputed model changes patched in where available. -use crate::planner_identity::_helpers::identities; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_identity::_helpers::identities::replace_with; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; /// Resolved actions with recomputed model changes patched in where available. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, resolved_actions, changes))] pub fn merge_recomputed_model_changes<'py>( resolved_actions: &Bound<'py, PyAny>, changes: &Bound<'py, PyAny>, ) -> PyResult> { - identities::merge_recomputed_model_changes(resolved_actions, changes) + let py: Python<'py> = resolved_actions.py(); + let recomputed: Bound<'py, PyAny> = native_field(changes, "models")?; + let merged: Bound<'py, PyDict> = PyDict::new(py); + for item in native_field(resolved_actions, "models")? + .call_method0("items")? + .try_iter()? + { + let (name, resolved): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let mut change: Bound<'py, PyAny> = mapping_get(&recomputed, &name)?; + if change.is_none() { + change = native_field(&resolved, "change")?; + } + merged.set_item(name, replace_with(&resolved, ("change", change))?)?; + } + replace_with(resolved_actions, ("models", merged.into_any())) } diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/model_declared_columns_hash.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/model_declared_columns_hash.rs index 2953fb0967..f957eaadd1 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/model_declared_columns_hash.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/model_declared_columns_hash.rs @@ -1,12 +1,45 @@ //! The hash of a model's own declared columns, column families and contract setting. -use crate::planner_identity::_helpers::metadata; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_identity::_helpers::metadata::column_signature; +use crate::planner_identity::_helpers::metadata::family_signatures; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods}; +use sqlbuild_analysis::planner_identity::constants::CONTRACT_CONFIG_KEY; +use sqlbuild_core::fingerprint_identity::main::stable_identity_hash::stable_identity_hash; +use sqlbuild_python_base::planner_identity::constants::COMPACT; +use sqlbuild_python_base::planner_identity::main::sorted_dumps::sorted_dumps; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; /// The hash of a model's own declared columns, column families and contract setting. -#[inline] pub fn model_declared_columns_hash(model: &Bound<'_, PyAny>) -> PyResult { - metadata::model_declared_columns_hash(model) + let py: Python<'_> = model.py(); + let schema_entry: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( + model, + )?; + let payload: Bound<'_, PyDict> = PyDict::new(py); + payload.set_item( + "contract", + config_value(py, &model_values(model)?, CONTRACT_CONFIG_KEY)?, + )?; + let columns: Bound<'_, PyList> = PyList::empty(py); + let mut families: Bound<'_, PyList> = PyList::empty(py); + let mut type_enforcement: Bound<'_, PyAny> = py.None().into_bound(py); + if !schema_entry.is_none() { + for column in native_field(&schema_entry, "columns")?.try_iter()? { + columns.append(column_signature(&column?)?)?; + } + families = family_signatures(&native_field(&schema_entry, "dynamic_columns")?)?; + type_enforcement = native_field(&schema_entry, "type_enforcement")?; + } + payload.set_item("columns", columns)?; + payload.set_item("dynamic_columns", families)?; + payload.set_item("type_enforcement", type_enforcement)?; + Ok(stable_identity_hash(&sorted_dumps( + payload.as_any(), + COMPACT, + true, + )?)) } diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/model_version_identity_metadata_json.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/model_version_identity_metadata_json.rs index 2d04303772..e36be6e0d3 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/model_version_identity_metadata_json.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/model_version_identity_metadata_json.rs @@ -1,20 +1,49 @@ //! Deterministic non-query model fingerprint metadata JSON. -use crate::planner_identity::_helpers::metadata; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_identity::_helpers::metadata::execution_signature; +use crate::planner_identity::_helpers::metadata::version_identity_metadata_json; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::planner_identity::constants::{TABLE_FN_RESOURCE, UDF_RESOURCE}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_project::compiled_views::main::compiled_models::config::config; +use sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; /// Deterministic non-query model fingerprint metadata JSON. -#[inline] pub fn model_version_identity_metadata_json<'py>( model: &Bound<'py, PyAny>, function_local_hashes: Option<&Bound<'py, PyAny>>, hook_version_hashes: Option<&Bound<'py, PyAny>>, ) -> PyResult { - metadata::model_version_identity_metadata_json( - model, - function_local_hashes, - hook_version_hashes, + let py: Python<'py> = model.py(); + let local_function_hashes: Bound<'py, PyDict> = PyDict::new(py); + if let Some(function_hashes) = function_local_hashes.filter(|hashes| !hashes.is_none()) { + for upstream_key in deps(model)?.try_iter()? { + let upstream_key: Bound<'py, PyAny> = upstream_key?; + let kind: String = native_field(&upstream_key, "resource_type")? + .str()? + .extract()?; + if kind != UDF_RESOURCE && kind != TABLE_FN_RESOURCE { + continue; + } + let name: Bound<'py, PyAny> = native_field(&upstream_key, "name")?; + let upstream_hash: Bound<'py, PyAny> = mapping_get(function_hashes, &name)?; + if !upstream_hash.is_none() { + local_function_hashes.set_item(name, upstream_hash)?; + } + } + } + let config_values: Bound<'py, PyAny> = native_field(&config(model)?, "values")?; + let signature: Bound<'py, PyDict> = + execution_signature(model, &config_values, hook_version_hashes)?; + version_identity_metadata_json( + &name_object(model)?, + &config_values, + ( + Some(local_function_hashes.into_any()), + Some(signature.into_any()), + ), ) } diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/or_empty.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/or_empty.rs index 16d965d69a..46bce02f05 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/or_empty.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/or_empty.rs @@ -1,16 +1,15 @@ //! `value or {}`. -use crate::planner_identity::_helpers::metadata; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; /// `value or {}`. -#[inline] pub fn or_empty<'py>( value: Option>, py: Python<'py>, ) -> PyResult> { - metadata::or_empty(value, py) + match value { + Some(value) if value.is_truthy()? => Ok(value), + _ => Ok(PyDict::new(py).into_any()), + } } diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/register_identities.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/register_identities.rs index f4e7192563..326afbe429 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/register_identities.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/register_identities.rs @@ -1,12 +1,15 @@ //! Register the identities functions and classes. -use crate::planner_identity::_helpers::identities; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_identity::_helpers::identities::build_direct_model_version_identities; +use crate::planner_identity::main::merge_recomputed_model_changes::merge_recomputed_model_changes; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the identities functions and classes. -#[inline] pub fn register_identities(module: &Bound<'_, PyModule>) -> PyResult<()> { - identities::register(module) + module.add_function(wrap_pyfunction!( + build_direct_model_version_identities, + module + )?)?; + module.add_function(wrap_pyfunction!(merge_recomputed_model_changes, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/register_metadata.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/register_metadata.rs index 3695c2f5b1..427c149ff5 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/register_metadata.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/register_metadata.rs @@ -1,12 +1,29 @@ //! Register the metadata functions and classes. -use crate::planner_identity::_helpers::metadata; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_identity::_helpers::metadata::build_version_identity_metadata_json; +use crate::planner_identity::_helpers::metadata::declared_columns_hash; +use crate::planner_identity::main::build_compiled_function_fingerprint_sql::build_compiled_function_fingerprint_sql; +use crate::planner_identity::main::build_function_local_hashes::build_function_local_hashes; +use crate::planner_identity::main::build_model_version_identity_metadata_json::build_model_version_identity_metadata_json; +use crate::planner_identity::main::build_seed_identity::build_seed_identity; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the metadata functions and classes. -#[inline] pub fn register_metadata(module: &Bound<'_, PyModule>) -> PyResult<()> { - metadata::register(module) + module.add_function(wrap_pyfunction!( + build_version_identity_metadata_json, + module + )?)?; + module.add_function(wrap_pyfunction!( + build_model_version_identity_metadata_json, + module + )?)?; + module.add_function(wrap_pyfunction!(declared_columns_hash, module)?)?; + module.add_function(wrap_pyfunction!(build_function_local_hashes, module)?)?; + module.add_function(wrap_pyfunction!( + build_compiled_function_fingerprint_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(build_seed_identity, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/register_staleness.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/register_staleness.rs index 41b0da1353..f65f88c5f7 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/register_staleness.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/register_staleness.rs @@ -1,12 +1,18 @@ //! Register the staleness functions and classes. -use crate::planner_identity::_helpers::staleness; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_identity::_helpers::staleness::classify_selection_staleness_warnings; +use crate::planner_identity::main::build_stale_out_of_selection_warnings::build_stale_out_of_selection_warnings; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the staleness functions and classes. -#[inline] pub fn register_staleness(module: &Bound<'_, PyModule>) -> PyResult<()> { - staleness::register(module) + module.add_function(wrap_pyfunction!( + build_stale_out_of_selection_warnings, + module + )?)?; + module.add_function(wrap_pyfunction!( + classify_selection_staleness_warnings, + module + )?) } diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/seed_identity.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/seed_identity.rs index b0fd0ed69f..b6d5ffa984 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/seed_identity.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/seed_identity.rs @@ -1,9 +1,7 @@ //! `(identity_hash, metadata_json)` of one compiled seed from its content and load-affecting config. use crate::planner_identity::_helpers::metadata; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// `(identity_hash, metadata_json)` of one compiled seed from its content and load-affecting config. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/with_honest_model_write_hashes.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/with_honest_model_write_hashes.rs index 15e1a01620..4f396f8310 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/with_honest_model_write_hashes.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/with_honest_model_write_hashes.rs @@ -1,10 +1,7 @@ //! Planned write hashes replaced by hashes based on upstreams available this run. use crate::planner_identity::_helpers::identities; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyDict; /// Planned write hashes replaced by hashes based on upstreams available this run. diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs index 423e54e922..244560de76 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs @@ -1,12 +1,8 @@ //! Gather live candidate evidence without changing adapter inspection ownership. -use crate::planner_migrations::_helpers::column::column_objects::config; -use crate::planner_migrations::_helpers::column::column_objects::definition_hash; -use crate::planner_migrations::_helpers::column::column_objects::lower; +use crate::planner_migrations::_helpers::column::column_objects::{config, definition_hash, lower}; use crate::planner_migrations::_helpers::projections::projections::parse_query_shape; -use crate::planner_migrations::models::ColumnCandidate; -use crate::planner_migrations::models::ColumnMigrationInputs; -use crate::planner_migrations::models::MigrationPhase; +use crate::planner_migrations::models::{ColumnCandidate, ColumnMigrationInputs, MigrationPhase}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs index f7307a2b22..18b11be9ee 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs @@ -3,15 +3,11 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::planner_migrations::_helpers::column::column_candidates::declared_names; -use crate::planner_migrations::_helpers::column::column_objects::lower; -use crate::planner_migrations::_helpers::column::column_objects::model; -use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_migrations::_helpers::column::column_objects::{lower, model, replace}; use crate::planner_migrations::_helpers::column::columns::identical_renames; use crate::planner_migrations::_helpers::column::columns::rename_hints; -use crate::planner_migrations::constants::AUTOMATIC; -use crate::planner_migrations::constants::DONE; -use crate::planner_migrations::models::CheckedColumns; -use crate::planner_migrations::models::ColumnCandidate; +use crate::planner_migrations::constants::{AUTOMATIC, DONE}; +use crate::planner_migrations::models::{CheckedColumns, ColumnCandidate}; use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs index fbf5822472..1b6155a88e 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs @@ -1,14 +1,11 @@ //! Project column renames onto the existing snapshot and redirect cursor reads. -use crate::planner_migrations::_helpers::column::column_objects::config; -use crate::planner_migrations::_helpers::column::column_objects::definition_hash; -use crate::planner_migrations::_helpers::column::column_objects::lower; -use crate::planner_migrations::_helpers::column::column_objects::native; -use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_migrations::_helpers::column::column_objects::{ + config, definition_hash, lower, native, replace, +}; use crate::planner_migrations::_helpers::projections::projections::renames_explain_change; use crate::planner_migrations::models::ColumnCandidate; -use crate::planner_migrations::models::ColumnMigrationInputs; -use crate::planner_migrations::models::MigrationPhase; +use crate::planner_migrations::models::{ColumnMigrationInputs, MigrationPhase}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::imported::imported; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs index 8d94e97b16..32d4707cb3 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs @@ -2,13 +2,13 @@ use crate::planner_migrations::_helpers::column::column_objects::replace; use crate::planner_migrations::_helpers::column::known_input_columns::KnownInputColumns; -use crate::planner_migrations::models::ColumnMigrationInputs; -use crate::planner_migrations::models::ColumnMigrationPlanning; -use crate::planner_migrations::models::MigrationPhase; +use crate::planner_migrations::models::{ + ColumnMigrationInputs, ColumnMigrationPlanning, MigrationPhase, +}; use crate::planner_phases::main::inputs::runtime_field::runtime_field; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PySet, PySetMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -19,7 +19,7 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::name_object: /// Decide column renames and return the existing planning result model. #[pyfunction(name = "plan_column_migrations")] #[pyo3(signature = (**options))] -fn plan_column_migrations_of<'py>( +pub(crate) fn plan_column_migrations_of<'py>( py: Python<'py>, options: Option>, ) -> PyResult<()> { @@ -156,7 +156,3 @@ pub(crate) fn plan_column_migrations<'py>( warnings, }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(plan_column_migrations_of, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs index 865c9af4f0..58375264e8 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs @@ -2,10 +2,10 @@ use crate::planner_migrations::_helpers::column::column_objects::native; use crate::planner_migrations::_helpers::column::column_objects::text; -use crate::planner_migrations::constants::DONE; -use crate::planner_migrations::constants::MISSING_ORIGIN_ALLOW_OUTCOME; -use crate::planner_migrations::constants::MISSING_ORIGIN_CONFIRMATION_OUTCOME; -use crate::planner_migrations::constants::MISSING_ORIGIN_DENY_OUTCOME; +use crate::planner_migrations::constants::{ + DONE, MISSING_ORIGIN_ALLOW_OUTCOME, MISSING_ORIGIN_CONFIRMATION_OUTCOME, + MISSING_ORIGIN_DENY_OUTCOME, +}; use pyo3::exceptions::PyKeyError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_migrations::main::column_warning::column_warning; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs index 2f55f97b27..87f78771c4 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs @@ -2,9 +2,9 @@ use std::collections::BTreeSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::planner_migrations::main::{ identical_renames as native_renames, rename_hints as native_hints, @@ -141,8 +141,3 @@ pub(crate) fn rename_hints<'py>( } PyTuple::new(py, result) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(identical_renames, module)?)?; - module.add_function(wrap_pyfunction!(rename_hints, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery.rs index d5fb030a4d..6f81614411 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery.rs @@ -3,11 +3,12 @@ use crate::planner_migrations::_helpers::discovery::discovery_events::renames; use crate::planner_migrations::_helpers::discovery::discovery_models::ordered; use crate::planner_migrations::_helpers::discovery::discovery_requests::automatic; -use crate::planner_migrations::_helpers::planning::matching::match_migration_fingerprints; -use crate::planner_migrations::_helpers::planning::matching::migration_destination_fingerprints; -use crate::planner_migrations::models::DiscoveryRequest; -use crate::planner_migrations::models::FingerprintInputs; -use crate::planner_migrations::models::ModelMigrationDiscovery; +use crate::planner_migrations::_helpers::planning::matching::{ + match_migration_fingerprints, migration_destination_fingerprints, +}; +use crate::planner_migrations::models::{ + DiscoveryRequest, FingerprintInputs, ModelMigrationDiscovery, +}; use pyo3::prelude::{PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple, PyTupleMethods}; use sqlbuild_python_base::boundary::main::call_with::call_with; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs index 5fde1c4193..48b937d2e1 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs @@ -2,12 +2,10 @@ use crate::planner_migrations::_helpers::column::column_objects::config; use crate::planner_migrations::_helpers::column::column_objects::native; -use crate::planner_migrations::_helpers::discovery::discovery_events::identity; -use crate::planner_migrations::_helpers::discovery::discovery_events::key; -use crate::planner_migrations::_helpers::discovery::discovery_events::newest; -use crate::planner_migrations::_helpers::discovery::discovery_events::relation; -use crate::planner_migrations::models::ModelMigrationDeclaration; -use crate::planner_migrations::models::ModelMigrationRequest; +use crate::planner_migrations::_helpers::discovery::discovery_events::{ + identity, key, newest, relation, +}; +use crate::planner_migrations::models::{ModelMigrationDeclaration, ModelMigrationRequest}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PySet, PySetMethods, PyString, PyTuple, diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs index 66f9710514..79828786c6 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs @@ -2,16 +2,12 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::planner_migrations::_helpers::column::column_objects::lower; -use crate::planner_migrations::_helpers::column::column_objects::model; -use crate::planner_migrations::_helpers::column::column_objects::replace; -use crate::planner_migrations::_helpers::column::column_objects::text; -use crate::planner_migrations::_helpers::model::model_handover::applies; -use crate::planner_migrations::_helpers::model::model_handover::empty_version; -use crate::planner_migrations::_helpers::model::model_handover::handover; +use crate::planner_migrations::_helpers::column::column_objects::{lower, model, replace, text}; +use crate::planner_migrations::_helpers::model::model_handover::{ + applies, empty_version, handover, +}; use crate::planner_migrations::_helpers::planning::compatibility::check_migration_compatibility; -use crate::planner_migrations::models::CompatibilityCheck; -use crate::planner_migrations::models::MigrationDecisionRequest; +use crate::planner_migrations::models::{CompatibilityCheck, MigrationDecisionRequest}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyBool; use sqlbuild_analysis::planner_migrations::main::model_decision::model_decision; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs index cc87ead457..e63588b412 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs @@ -1,7 +1,6 @@ //! Adapt immutable migration fingerprints without changing their persisted representation. -use crate::planner_migrations::_helpers::column::column_objects::replace; -use crate::planner_migrations::_helpers::column::column_objects::text; +use crate::planner_migrations::_helpers::column::column_objects::{replace, text}; use crate::planner_migrations::constants::MIGRATION_MODEL_NAME_METADATA_KEY; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs index 01a95ea529..709519a2ca 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs @@ -1,8 +1,7 @@ //! Project inherited fingerprints and live relation facts before normal planning. -use crate::planner_migrations::_helpers::column::column_objects::config; use crate::planner_migrations::_helpers::column::column_objects::native; -use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_migrations::_helpers::column::column_objects::{config, replace}; use crate::planner_migrations::models::MigrationPhase; use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs index ef56d35e05..31cb8c9989 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs @@ -1,18 +1,15 @@ //! Model migration orchestration over Python-owned live inspection. use crate::planner_changes::main::plan_reference_renames::plan_reference_renames; -use crate::planner_migrations::_helpers::column::column_objects::config; -use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_migrations::_helpers::column::column_objects::{config, replace}; use crate::planner_migrations::_helpers::discovery::discovery::discover_model_migrations; -use crate::planner_migrations::_helpers::planning::planning_origin::require_same_database; -use crate::planner_migrations::_helpers::planning::planning_origin::resolve_origin; -use crate::planner_migrations::models::DiscoveryRequest; -use crate::planner_migrations::models::MigrationDecisionRequest; -use crate::planner_migrations::models::MigrationPhase; -use crate::planner_migrations::models::ModelMigrationDeclaration; -use crate::planner_migrations::models::ModelMigrationDiscovery; -use crate::planner_migrations::models::ModelMigrationPlanning; -use crate::planner_migrations::models::ModelMigrationRequest; +use crate::planner_migrations::_helpers::planning::planning_origin::{ + require_same_database, resolve_origin, +}; +use crate::planner_migrations::models::{ + DiscoveryRequest, MigrationDecisionRequest, MigrationPhase, ModelMigrationDeclaration, + ModelMigrationDiscovery, ModelMigrationPlanning, ModelMigrationRequest, +}; use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyString, PyTuple}; @@ -80,24 +77,7 @@ fn manual_migration_requests<'py>( Ok(requests) } -/// Model migrations of the selected scope: discovered and declared moves, decided against live state. -pub(crate) fn plan_model_migrations<'py>( - phase: &MigrationPhase<'py>, - fingerprints: &Bound<'py, PyAny>, -) -> PyResult> { - let runtime = phase.runtime.clone(); - let snapshot = phase.snapshot.clone(); - let schemas = - crate::planner_migrations::_helpers::planning::planning_origin::project_schemas(&runtime)?; - let state = crate::planner_migrations::_helpers::discovery::state_inspection::inspection( - &runtime, - Some(&native_field(&snapshot, "fingerprints")?), - )?; - let discovery = discover((phase, fingerprints), &state, &schemas)?; - finish(phase, &state, &schemas, &discovery) -} - -fn discover<'py>( +pub(crate) fn discover<'py>( (phase, fingerprints): (&MigrationPhase<'py>, &Bound<'py, PyAny>), state: &Bound<'py, PyAny>, schemas: &Bound<'py, PyAny>, @@ -140,7 +120,7 @@ fn discover<'py>( }) } -fn finish<'py>( +pub(crate) fn finish<'py>( phase: &MigrationPhase<'py>, state: &Bound<'py, PyAny>, schemas: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs index 74223c718a..c29e3a9b4f 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs @@ -1,7 +1,8 @@ //! Adapt diagnostic display facts while preserving policy and code ordering. -use crate::planner_migrations::_helpers::column::column_warnings::missing_origin_outcome; -use crate::planner_migrations::_helpers::column::column_warnings::warning; +use crate::planner_migrations::_helpers::column::column_warnings::{ + missing_origin_outcome, warning, +}; use crate::planner_migrations::constants::HIDDEN_ORIGIN_REMEDY; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyString; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs index d220965f4a..18f8f935b7 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs @@ -1,16 +1,12 @@ //! Assemble new and resumed old-name view steps from recorded facts. -use crate::planner_migrations::_helpers::column::column_objects::model; -use crate::planner_migrations::_helpers::column::column_objects::text; -use crate::planner_migrations::_helpers::old_name::old_name_objects::expires; -use crate::planner_migrations::_helpers::old_name::old_name_objects::location; -use crate::planner_migrations::_helpers::old_name::old_name_objects::relation; -use crate::planner_migrations::_helpers::old_name::old_name_objects::retention; -use crate::planner_migrations::_helpers::old_name::old_name_objects::stores_history; +use crate::planner_migrations::_helpers::column::column_objects::{model, text}; +use crate::planner_migrations::_helpers::old_name::old_name_objects::{ + expires, location, relation, retention, stores_history, +}; use crate::planner_migrations::_helpers::old_name::old_names::declared_column_aliases; use crate::planner_migrations::_helpers::old_name::old_names::old_name_view_skip_reason; -use crate::planner_migrations::constants::PENDING_ARCHIVE; -use crate::planner_migrations::constants::PENDING_VIEW; +use crate::planner_migrations::constants::{PENDING_ARCHIVE, PENDING_VIEW}; use crate::planner_migrations::models::OldNamePlanner; use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs index 5eb67253db..7518d1ea03 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs @@ -2,8 +2,7 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::planner_migrations::_helpers::column::column_objects::config; -use crate::planner_migrations::_helpers::column::column_objects::model; +use crate::planner_migrations::_helpers::column::column_objects::{config, model}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs index 871c2e9c66..5f591d5791 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs @@ -5,8 +5,7 @@ use sqlbuild_host::entries::models::AdapterHost; use crate::planner_migrations::_helpers::old_name::old_name_objects::recorded_view; use crate::planner_migrations::_helpers::old_name::old_names::old_name_claim_errors; use crate::planner_migrations::_helpers::old_name::old_names::old_name_schemas_to_read; -use crate::planner_migrations::models::MigrationPhase; -use crate::planner_migrations::models::OldNameViewPlanning; +use crate::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PySet, PySetMethods, PyTuple}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs index 8b0e57d3f1..4607ee3b2d 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs @@ -2,8 +2,7 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::planner_migrations::_helpers::column::column_objects::native; -use crate::planner_migrations::_helpers::column::column_objects::text; +use crate::planner_migrations::_helpers::column::column_objects::{native, text}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/compatibility.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/compatibility.rs index afcaf2a97d..37c9a370ea 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/compatibility.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/compatibility.rs @@ -1,8 +1,7 @@ //! Convert existing planner contracts for origin compatibility checks. use crate::planner_changes::main::detect_model_changes::detect_model_changes; -use crate::planner_migrations::models::CompatibilityCheck; -use crate::planner_migrations::models::MigrationCompatibilityResult; +use crate::planner_migrations::models::{CompatibilityCheck, MigrationCompatibilityResult}; use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs index 13d8a64ed0..26d7df98bf 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs @@ -1,7 +1,6 @@ //! Fingerprint cache callbacks around native one-to-one migration matching. -use crate::planner_migrations::models::FingerprintInputs; -use crate::planner_migrations::models::ModelMigrationRequest; +use crate::planner_migrations::models::{FingerprintInputs, ModelMigrationRequest}; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs index 53d4a0e078..7b08211617 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs @@ -1,15 +1,10 @@ //! Migration identity binding with unchanged metadata and hash format. -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::planner_migrations::main::{migration_fingerprint, migration_ref_names}; +use crate::planner_migrations::main::fingerprint::fingerprint; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; -use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; -use sqlbuild_python_project::native_host::models::ModelLookup; use std::collections::BTreeMap; /// Hash normalized SQL and non-storage metadata through rename identities. @@ -45,74 +40,3 @@ pub(crate) fn build_migration_fingerprint( dialect.as_deref(), ) } - -/// The migration fingerprint of SQL and metadata through rename identities, `generic` by default. -pub(crate) fn fingerprint( - sql: &str, - metadata: &str, - identities: &BTreeMap, - dialect: Option<&str>, -) -> PyResult> { - migration_fingerprint::migration_fingerprint( - sql, - metadata, - identities, - dialect - .filter(|value| !value.is_empty()) - .unwrap_or("generic"), - ) - .map_err(PyValueError::new_err) -} - -/// Every name whose rename identity can change the migration fingerprint. -pub(crate) fn migration_fingerprint_ref_names<'py>( - py: Python<'py>, - query_sql: &str, - metadata_json: &str, -) -> PyResult> { - PyFrozenSet::new( - py, - migration_ref_names::migration_ref_names(query_sql, metadata_json) - .map_err(PyValueError::new_err)?, - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_migration_fingerprint, module)?)?; - Ok(()) -} - -/// Attach persisted migration identities through the existing cache boundary. -pub(crate) fn with_migration_fingerprints<'py>( - entries: &Bound<'py, PyAny>, - models: &ModelLookup, - cache: &Bound<'py, PyAny>, - dialect: &Bound<'py, PyAny>, -) -> PyResult> { - use sqlbuild_python_base::boundary::main::call_with::call_with; - let py = entries.py(); - let mut attached: Vec> = Vec::new(); - for entry in entries.try_iter()? { - let entry = entry?; - let model = models.get(py, &native_field(&entry, "name")?.extract::()?)?; - let metadata = native_field(&entry, "fingerprint_metadata_json")?; - if model.is_none() || metadata.is_none() || native_field(&entry, "action")?.eq("skip")? { - attached.push(entry); - continue; - } - let fingerprint = call_with( - &native_field(cache, "fingerprint")?, - &[ - ("query_sql", query_sql(&model)?), - ("metadata_json", metadata), - ("ref_identities", PyDict::new(py).into_any()), - ("dialect", dialect.clone()), - ], - )?; - let kwargs = PyDict::new(py); - kwargs.set_item("migration_fingerprint", fingerprint)?; - attached.push(replaced_keywords(&entry, &kwargs)?); - } - cache.call_method0("persist")?; - PyTuple::new(py, attached) -} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/projections.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/projections.rs index 63a466f24c..92db2b5325 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/projections.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/projections.rs @@ -2,9 +2,9 @@ use crate::planner_migrations::errors::ProofError; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::planner_migrations::main::{query_shape, renames_explain_change as proof}; use sqlbuild_analysis::planner_migrations::models::{QueryProjection, QueryShape}; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -146,8 +146,3 @@ pub(crate) fn renames_explain_change( &arguments.value("input_columns")?, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_query_shape, module)?)?; - module.add_function(wrap_pyfunction!(renames_explain_change, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/fingerprint.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/fingerprint.rs index 976920e395..24dcffc7e1 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/main/fingerprint.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/fingerprint.rs @@ -1,16 +1,24 @@ //! The migration fingerprint of SQL and metadata through rename identities, `generic` by default. -use crate::planner_migrations::_helpers::projections::fingerprint; +use pyo3::exceptions::PyValueError; use pyo3::prelude::PyResult; +use sqlbuild_analysis::planner_migrations::main::migration_fingerprint; use std::collections::BTreeMap; /// The migration fingerprint of SQL and metadata through rename identities, `generic` by default. -#[inline] pub fn fingerprint( sql: &str, metadata: &str, identities: &BTreeMap, dialect: Option<&str>, ) -> PyResult> { - fingerprint::fingerprint(sql, metadata, identities, dialect) + migration_fingerprint::migration_fingerprint( + sql, + metadata, + identities, + dialect + .filter(|value| !value.is_empty()) + .unwrap_or("generic"), + ) + .map_err(PyValueError::new_err) } diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/migration_fingerprint_ref_names.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/migration_fingerprint_ref_names.rs index 5d53694370..258fc977fc 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/main/migration_fingerprint_ref_names.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/migration_fingerprint_ref_names.rs @@ -1,17 +1,19 @@ //! Every name whose rename identity can change the migration fingerprint. -use crate::planner_migrations::_helpers::projections::fingerprint; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyResult, Python}; use pyo3::types::PyFrozenSet; +use sqlbuild_analysis::planner_migrations::main::migration_ref_names; /// Every name whose rename identity can change the migration fingerprint. -#[inline] pub fn migration_fingerprint_ref_names<'py>( py: Python<'py>, query_sql: &str, metadata_json: &str, ) -> PyResult> { - fingerprint::migration_fingerprint_ref_names(py, query_sql, metadata_json) + PyFrozenSet::new( + py, + migration_ref_names::migration_ref_names(query_sql, metadata_json) + .map_err(PyValueError::new_err)?, + ) } diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_column_migrations.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_column_migrations.rs index 6f49497215..201f729356 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_column_migrations.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_column_migrations.rs @@ -1,9 +1,9 @@ //! Decide column renames for the selected incremental and snapshot models. use crate::planner_migrations::_helpers::column::column_planning; -use crate::planner_migrations::models::ColumnMigrationInputs; -use crate::planner_migrations::models::ColumnMigrationPlanning; -use crate::planner_migrations::models::MigrationPhase; +use crate::planner_migrations::models::{ + ColumnMigrationInputs, ColumnMigrationPlanning, MigrationPhase, +}; use pyo3::prelude::PyResult; /// Decide column renames for the selected incremental and snapshot models. diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_model_migrations.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_model_migrations.rs index 21dfde2f07..2409dc6fe7 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_model_migrations.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_model_migrations.rs @@ -1,17 +1,24 @@ //! Model migrations of the selected scope: discovered and declared moves, decided against live state. -use crate::planner_migrations::_helpers::model::model_planning; -use crate::planner_migrations::models::MigrationPhase; -use crate::planner_migrations::models::ModelMigrationPlanning; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_migrations::_helpers::model::model_planning::discover; +use crate::planner_migrations::_helpers::model::model_planning::finish; +use crate::planner_migrations::models::{MigrationPhase, ModelMigrationPlanning}; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Model migrations of the selected scope: discovered and declared moves, decided against live state. -#[inline] pub fn plan_model_migrations<'py>( phase: &MigrationPhase<'py>, fingerprints: &Bound<'py, PyAny>, ) -> PyResult> { - model_planning::plan_model_migrations(phase, fingerprints) + let runtime = phase.runtime.clone(); + let snapshot = phase.snapshot.clone(); + let schemas = + crate::planner_migrations::_helpers::planning::planning_origin::project_schemas(&runtime)?; + let state = crate::planner_migrations::_helpers::discovery::state_inspection::inspection( + &runtime, + Some(&native_field(&snapshot, "fingerprints")?), + )?; + let discovery = discover((phase, fingerprints), &state, &schemas)?; + finish(phase, &state, &schemas, &discovery) } diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_old_name_views.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_old_name_views.rs index a5ac9ef575..4388949a57 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_old_name_views.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_old_name_views.rs @@ -1,11 +1,8 @@ //! Old-name compatibility views to create, resume or retire for the planned model and column migrations. use crate::planner_migrations::_helpers::old_name::old_name_planning; -use crate::planner_migrations::models::MigrationPhase; -use crate::planner_migrations::models::OldNameViewPlanning; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Old-name compatibility views to create, resume or retire for the planned model and column migrations. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_column_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_column_planning.rs index cff40e933b..6cc5b02f6c 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_column_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_column_planning.rs @@ -1,12 +1,10 @@ //! Register the column planning functions and classes. -use crate::planner_migrations::_helpers::column::column_planning; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_migrations::_helpers::column::column_planning::plan_column_migrations_of; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the column planning functions and classes. -#[inline] pub fn register_column_planning(module: &Bound<'_, PyModule>) -> PyResult<()> { - column_planning::register(module) + module.add_function(wrap_pyfunction!(plan_column_migrations_of, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_columns.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_columns.rs index f1bcf4f3f1..6d88a83b9b 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_columns.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_columns.rs @@ -1,12 +1,12 @@ //! Register the columns functions and classes. -use crate::planner_migrations::_helpers::column::columns; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_migrations::_helpers::column::columns::identical_renames; +use crate::planner_migrations::_helpers::column::columns::rename_hints; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the columns functions and classes. -#[inline] pub fn register_columns(module: &Bound<'_, PyModule>) -> PyResult<()> { - columns::register(module) + module.add_function(wrap_pyfunction!(identical_renames, module)?)?; + module.add_function(wrap_pyfunction!(rename_hints, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_fingerprint.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_fingerprint.rs index 3bd9e3d38b..f2477f5ceb 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_fingerprint.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_fingerprint.rs @@ -1,12 +1,11 @@ //! Register the fingerprint functions and classes. -use crate::planner_migrations::_helpers::projections::fingerprint; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_migrations::_helpers::projections::fingerprint::build_migration_fingerprint; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the fingerprint functions and classes. -#[inline] pub fn register_fingerprint(module: &Bound<'_, PyModule>) -> PyResult<()> { - fingerprint::register(module) + module.add_function(wrap_pyfunction!(build_migration_fingerprint, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_projections.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_projections.rs index f6a8f4a0ca..5d3d1d9648 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_projections.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_projections.rs @@ -1,12 +1,12 @@ //! Register the projections functions and classes. -use crate::planner_migrations::_helpers::projections::projections; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_migrations::_helpers::projections::projections::parse_query_shape; +use crate::planner_migrations::_helpers::projections::projections::renames_explain_change; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the projections functions and classes. -#[inline] pub fn register_projections(module: &Bound<'_, PyModule>) -> PyResult<()> { - projections::register(module) + module.add_function(wrap_pyfunction!(parse_query_shape, module)?)?; + module.add_function(wrap_pyfunction!(renames_explain_change, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs index 22c55b847b..6df24cdd66 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs @@ -1,19 +1,43 @@ //! Attach persisted migration identities through the existing cache boundary. -use crate::planner_migrations::_helpers::projections::fingerprint; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyTuple; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; use sqlbuild_python_project::native_host::models::ModelLookup; /// Attach persisted migration identities through the existing cache boundary. -#[inline] pub fn with_migration_fingerprints<'py>( entries: &Bound<'py, PyAny>, models: &ModelLookup, cache: &Bound<'py, PyAny>, dialect: &Bound<'py, PyAny>, ) -> PyResult> { - fingerprint::with_migration_fingerprints(entries, models, cache, dialect) + use sqlbuild_python_base::boundary::main::call_with::call_with; + let py = entries.py(); + let mut attached: Vec> = Vec::new(); + for entry in entries.try_iter()? { + let entry = entry?; + let model = models.get(py, &native_field(&entry, "name")?.extract::()?)?; + let metadata = native_field(&entry, "fingerprint_metadata_json")?; + if model.is_none() || metadata.is_none() || native_field(&entry, "action")?.eq("skip")? { + attached.push(entry); + continue; + } + let fingerprint = call_with( + &native_field(cache, "fingerprint")?, + &[ + ("query_sql", query_sql(&model)?), + ("metadata_json", metadata), + ("ref_identities", PyDict::new(py).into_any()), + ("dialect", dialect.clone()), + ], + )?; + let kwargs = PyDict::new(py); + kwargs.set_item("migration_fingerprint", fingerprint)?; + attached.push(replaced_keywords(&entry, &kwargs)?); + } + cache.call_method0("persist")?; + PyTuple::new(py, attached) } diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/models.rs b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs index 9a11795015..f188887cb0 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs @@ -9,9 +9,7 @@ use crate::planner_migrations::_helpers::discovery::state_inspection::schema_or_ use crate::planner_migrations::_helpers::discovery::state_inspection::tuple_of; use crate::planner_migrations::constants::GET_COLUMNS_FOR_INSPECTION; use crate::planner_migrations::constants::LIST_RELATIONS_FOR_INSPECTION; -use pyo3::prelude::Bound; use pyo3::prelude::Py; -use pyo3::prelude::PyAny; use pyo3::prelude::PyAnyMethods; use pyo3::prelude::PyRef; use pyo3::prelude::PyRefMut; @@ -19,6 +17,7 @@ use pyo3::prelude::PyResult; use pyo3::prelude::Python; use pyo3::prelude::pyclass; use pyo3::prelude::pymethods; +use pyo3::prelude::{Bound, PyAny}; use pyo3::types::PyBool; use pyo3::types::PyDict; use pyo3::types::PyString; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs index 3bbbc018fd..48d89d4596 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs @@ -3,9 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::planner_phases::models::PlannerIdentityContext; -use crate::planner_phases::models::PlannerRuntime; -use crate::planner_phases::models::PlannerWarehouseState; +use crate::planner_phases::models::{ + PlannerIdentityContext, PlannerRuntime, PlannerWarehouseState, +}; use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use sqlbuild_python_project::planner_scope::models::RetainedScope; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs index cf1926333b..5b3535f9a9 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs @@ -5,8 +5,7 @@ use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; use pyo3::types::{PyDict, PyTuple}; -use crate::planner_phases::models::PlannerRuntime; -use crate::planner_phases::models::PlannerWarehouseState; +use crate::planner_phases::models::{PlannerRuntime, PlannerWarehouseState}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::build_command::main::utc_now::utc_now; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs index 07ebe967f3..422b8bdbc9 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs @@ -5,11 +5,10 @@ use crate::planner_entries::models::RelationsRequest; use crate::planner_migrations::main::plan_column_migrations::plan_column_migrations; use crate::planner_migrations::main::plan_model_migrations::plan_model_migrations; use crate::planner_migrations::main::plan_old_name_views::plan_old_name_views as plan_old_name_migration_views; -use crate::planner_migrations::models::ColumnMigrationInputs; -use crate::planner_migrations::models::MigrationPhase; +use crate::planner_migrations::models::{ColumnMigrationInputs, MigrationPhase}; use crate::planner_phases::_helpers::inputs::contexts::class_field; use crate::planner_phases::_helpers::inputs::runtime::runtime_field; -use crate::planner_phases::_helpers::rebuilds::full_refresh::effectively_full_refreshed_model_names; +use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; use crate::planner_phases::models::PlannerWarehouseState; use crate::planner_warehouse::main::inspection::warehouse_snapshot::warehouse_snapshot; use crate::planner_warehouse::models::MigrationFingerprintCache; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs index d0d8f1a8b0..5e5c90a152 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs @@ -47,7 +47,7 @@ pub(crate) fn planner_identity_work_info( Ok(info) } -fn version_identities<'py>( +pub(crate) fn version_identities<'py>( project: &Bound<'py, PyAny>, scope: Bound<'py, PyAny>, recorded_fingerprints: &Bound<'py, PyAny>, @@ -65,36 +65,6 @@ fn version_identities<'py>( ) } -pub(crate) fn build_planner_identity_context<'py>( - project: &Bound<'py, PyAny>, - (scopes, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - include_stale_warning_identities: bool, -) -> PyResult> { - let recorded_fingerprints: Bound<'py, PyAny> = native_field(snapshot, "fingerprints")?; - let identities = version_identities( - project, - class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, - &recorded_fingerprints, - )?; - let stale = if include_stale_warning_identities { - version_identities( - project, - class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)?, - &recorded_fingerprints, - )? - } else { - identities.clone() - }; - Ok(Bound::new( - project.py(), - PlannerIdentityContext { - version_identities: identities.unbind(), - stale_warning_identities: stale.unbind(), - }, - )? - .into_any()) -} - fn reusable_model_names( identities: &Bound<'_, PyAny>, execution_changes: &Bound<'_, PyAny>, @@ -217,17 +187,3 @@ pub(crate) fn detect_stale_warning_changes<'py>( ], ) } - -/// `detect_stale_warning_changes_of` as a function of `module`. -pub(crate) fn detect_stale_warning_changes_of_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(detect_stale_warning_changes_of, module) -} - -/// `planner_identity_work_info` as a function of `module`. -pub(crate) fn planner_identity_work_info_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(planner_identity_work_info, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs index fc26ab100b..f3c26b7b1f 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs @@ -7,20 +7,16 @@ use crate::planner_migrations::main::with_migration_fingerprints::with_migration use crate::planner_phases::_helpers::inputs::contexts::class_field; use crate::planner_phases::_helpers::inputs::runtime::runtime_field; use crate::planner_phases::_helpers::inputs::runtime::warehouse_field; -use crate::planner_phases::_helpers::rebuilds::retention::plan_retention; -use crate::planner_phases::_helpers::rebuilds::retention::plan_table_types; -use crate::planner_phases::models::PlanOutputContext; -use crate::planner_phases::models::PlanOutputFields; -use crate::planner_phases::models::PlanWarningContext; -use crate::planner_phases::models::PlannedModelEntries; +use crate::planner_phases::_helpers::rebuilds::retention::{plan_retention, plan_table_types}; use crate::planner_phases::models::PlannerIdentityContext; -use crate::planner_phases::models::StoragePolicyContext; +use crate::planner_phases::models::{ + PlanOutputContext, PlanOutputFields, PlanWarningContext, PlannedModelEntries, + StoragePolicyContext, +}; use crate::planner_warehouse::main::sources::source_freshness_unknown_warnings::source_freshness_unknown_warnings; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; -use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; @@ -31,12 +27,14 @@ use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_phases::main::replace::replace; use sqlbuild_python_project::planner_phases::main::sorted_names::sorted_names; use sqlbuild_python_project::planner_phases::main::strings::strings; -use sqlbuild_python_project::planner_phases::main::text::text; use sqlbuild_python_project::planner_phases::main::tuple::tuple; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; -fn concat<'py>(py: Python<'py>, parts: &[Bound<'py, PyAny>]) -> PyResult> { +pub(crate) fn concat<'py>( + py: Python<'py>, + parts: &[Bound<'py, PyAny>], +) -> PyResult> { let mut items: Vec<_> = Vec::new(); for part in parts { for item in part.try_iter()? { @@ -47,7 +45,7 @@ fn concat<'py>(py: Python<'py>, parts: &[Bound<'py, PyAny>]) -> PyResult PlanOutputFields<'py> { - fn field(&self, name: &str) -> PyResult> { + pub(crate) fn field(&self, name: &str) -> PyResult> { self.fields .iter() .find(|(field, _)| *field == name) @@ -55,7 +53,7 @@ impl<'py> PlanOutputFields<'py> { .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(name.to_owned())) } - fn set(&mut self, name: &'static str, value: Bound<'py, PyAny>) { + pub(crate) fn set(&mut self, name: &'static str, value: Bound<'py, PyAny>) { match self.fields.iter_mut().find(|(field, _)| *field == name) { Some(existing) => existing.1 = value, None => self.fields.push((name, value)), @@ -287,85 +285,6 @@ pub(crate) fn add_plan_warnings<'py>( Ok(output) } -/// The unselected models the selected scope reads, sorted. -pub(crate) fn unselected_upstream_models(scopes: &Bound<'_, PyAny>) -> PyResult> { - let selected = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; - let selected = sqlbuild_python_project::planner_scope::main::scope_data::scope_data(&selected)?; - let mut names: Vec = Vec::new(); - for (key, deps) in &selected.upstream { - if !selected.selected.contains(key) { - continue; - } - for dep in deps { - if dep.0 == MODEL && !selected.selected.contains(dep) { - names.push(dep.1.clone()); - } - } - } - names.sort(); - names.dedup(); - Ok(names) -} - -/// Note unselected upstreams deferred to pre-migration names, and each lookup in `failures`. -pub(crate) fn add_deferred_origin_notes<'py>( - mut output: PlanOutputFields<'py>, - (project, scopes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - (deferral, failures): (&Bound<'py, PyAny>, &[(String, String)]), -) -> PyResult> { - let py = project.py(); - let locations = native_field(deferral, "deferred_locations")?; - if locations.is_none() { - return Ok(output); - } - let names: Vec = unselected_upstream_models(scopes)?; - let warning = |name: &str, message: &str| -> PyResult> { - native( - py, - RecordId::PlanWarning, - &[ - ("model_name", text(py, name)), - ( - "severity", - enum_member(py, RecordId::WarningSeverity, "WARNING")?, - ), - ("message", text(py, message)), - ], - ) - }; - let mut notes: Vec> = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { - let model = model?; - let name: String = native_field(&model, "name")?.extract()?; - if names.binary_search(&name).is_err() { - continue; - } - let Some(location) = locations.downcast::()?.get_item(&name)? else { - continue; - }; - if native_field(&location, "name")? - .eq(native_field(&native_field(&model, "destination")?, "name")?)? - { - continue; - } - notes.push(warning( - &name, - &format!( - "deferred to pre-migration name {}: the deferred target has not built model '{name}' under its current name yet", - native_field(&location, "qualified_name")? - ), - )?); - } - for (name, message) in failures { - notes.push(warning(name, message)?); - } - if !notes.is_empty() { - let warnings = concat(py, &[output.field("warnings")?, tuple(py, notes)?])?; - output.set("warnings", warnings); - } - Ok(output) -} - fn source_names<'py>(items: &Bound<'py, PyAny>) -> PyResult> { let mut names: Vec<_> = Vec::new(); for item in items.try_iter()? { @@ -385,7 +304,9 @@ fn age_names<'py>(statuses: &Bound<'py, PyAny>, wanted: &str) -> PyResult(source_freshness: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn freshness_metadata<'py>( + source_freshness: &Bound<'py, PyAny>, +) -> PyResult> { let py = source_freshness.py(); let propagation = native_field(source_freshness, "propagation")?; let propagated = |field: &str| -> PyResult> { @@ -437,27 +358,3 @@ fn freshness_metadata<'py>(source_freshness: &Bound<'py, PyAny>) -> PyResult( - mut output: PlanOutputFields<'py>, - source_freshness: &Bound<'py, PyAny>, - policies: &Bound<'py, PyAny>, -) -> PyResult> { - let py = source_freshness.py(); - let diagnostics = PyDict::new(py); - diagnostics.set_item("mode", text(py, "direct"))?; - diagnostics.set_item("enabled", native_field(policies, "selection_diagnostics")?)?; - output.metadata.push(( - "direct_source_freshness", - freshness_metadata(source_freshness)?, - )); - output.metadata.push(( - "direct_remaining_stale_model_names", - sorted_names(py, Vec::new())?, - )); - output - .metadata - .push(("selection_diagnostics", diagnostics.into_any())); - Ok(output) -} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs index b2394aa97c..582a32a21f 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs @@ -3,55 +3,15 @@ use crate::planner_entries::main::entries::default_microbatch_limit_action::default_microbatch_limit_action; use crate::planner_entries::main::entries::plan_entries::plan_entries; use crate::planner_entries::main::entries::planner_relations_context::planner_relations_context; -use crate::planner_entries::models::PlanEntriesRequest; -use crate::planner_entries::models::PlanEntryBuildInputs; -use crate::planner_entries::models::RelationsRequest; -use crate::planner_identity::main::merge_recomputed_model_changes::merge_recomputed_model_changes; -use crate::planner_identity::main::with_honest_model_write_hashes::with_honest_model_write_hashes; -use crate::planner_phases::_helpers::inputs::contexts::class_field; +use crate::planner_entries::models::{PlanEntriesRequest, PlanEntryBuildInputs, RelationsRequest}; use crate::planner_phases::_helpers::inputs::runtime::runtime_field; use crate::planner_phases::_helpers::inputs::runtime::warehouse_field; -use crate::planner_phases::models::ChangeReconciliation; -use crate::planner_phases::models::ModelEntryResults; -use crate::planner_phases::models::PlannerIdentityContext; +use crate::planner_phases::models::{ChangeReconciliation, ModelEntryResults}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyFrozenSet; -use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::command_runtime::main::resolve_configured_concurrency::resolve_configured_concurrency; -/// The execution scope's changes with honest write hashes, and the actions merged from them. -pub(crate) fn reconcile_execution_changes<'py>( - (execution_scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - identities: &Bound<'py, PyAny>, - resolved_actions: &Bound<'py, PyAny>, - changes: &Bound<'py, PyAny>, -) -> PyResult> { - let py = execution_scope.py(); - let honest = with_honest_model_write_hashes( - py, - Some(keywords( - py, - &[ - ("scope", execution_scope.clone()), - ("snapshot", snapshot.clone()), - ("changes", changes.clone()), - ( - "version_identities", - class_field(identities, |i: &PlannerIdentityContext| { - &i.version_identities - })?, - ), - ], - )?), - )?; - let resolved = merge_recomputed_model_changes(resolved_actions, &honest)?; - Ok(ChangeReconciliation { - changes: honest, - resolved_actions: resolved, - }) -} - fn config_or<'py>( config: &Bound<'py, PyAny>, path: &[&str], diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/full_refresh.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/full_refresh.rs index 35ac7b4cb1..dd3b389340 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/full_refresh.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/full_refresh.rs @@ -1,112 +1,13 @@ //! Per-model full-refresh resolution and full-rebuild refusals. -use crate::planner_phases::constants::FULL_REFRESH_DISABLED_REBUILD_CODE; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::pyfunction; -use pyo3::types::PyBool; -use sqlbuild_analysis::planner_warehouse::main::full_rebuild_cause::full_rebuild_cause; -use sqlbuild_analysis::planner_warehouse::models::FullRebuildFacts; -use sqlbuild_python_base::records::main::fields::field::field as native_field; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; -use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; -use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; -use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; -use sqlbuild_python_project::planner_phases::main::strings::strings; const FULL_REFRESH_KEY: &str = "full_refresh"; -fn configured_full_refresh<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - config_value(model.py(), &model_values(model)?, FULL_REFRESH_KEY) -} - -/// dbt-compatible nullable model full-refresh semantics. -pub(crate) fn resolve_model_full_refresh( - model: &Bound<'_, PyAny>, - cli_full_refresh: bool, -) -> PyResult { - let configured = configured_full_refresh(model)?; - if configured.is_instance_of::() { - return configured.extract(); - } - Ok(cli_full_refresh) -} - -/// Names of models effectively full refreshed for this invocation. -pub(crate) fn effectively_full_refreshed_model_names<'py>( - project: &Bound<'py, PyAny>, - cli_full_refresh: bool, +pub(crate) fn configured_full_refresh<'py>( + model: &Bound<'py, PyAny>, ) -> PyResult> { - let mut names: Vec<_> = Vec::new(); - for model in compiled_models(project)?.try_iter()? { - let model = model?; - if resolve_model_full_refresh(&model, cli_full_refresh)? { - names.push(name_object(&model)?); - } - } - frozenset(project.py(), &names) -} - -#[pyfunction] -#[pyo3(signature = (*, change_result, full_refresh))] -pub(crate) fn describe_full_rebuild_cause( - change_result: &Bound<'_, PyAny>, - full_refresh: bool, -) -> PyResult { - Ok(full_rebuild_cause(&FullRebuildFacts { - full_refresh, - change_kind: native_field(change_result, "change_kind")? - .str()? - .extract::()?, - query_changed: native_field(change_result, "query_changed")?.is_truthy()?, - changed_functions: strings(&native_field(change_result, "changed_functions")?)?, - has_schema_findings: native_field(change_result, "schema_findings")?.is_truthy()?, - })) -} - -#[pyfunction] -#[pyo3(signature = (*, model, change_result, full_rebuild_cause))] -pub(crate) fn check_full_rebuild_allowed( - model: &Bound<'_, PyAny>, - change_result: &Bound<'_, PyAny>, - full_rebuild_cause: &str, -) -> PyResult<()> { - let py = model.py(); - if !configured_full_refresh(model)?.is(PyBool::new(py, false)) { - return Ok(()); - } - if native_field(change_result, "change_kind")?.eq("first_run")? - || native_field(change_result, "query_changed")?.is_truthy()? - { - return Ok(()); - } - let name: String = model_name(model)?; - Err(planner_error( - py, - &format!( - "model '{name}' sets full_refresh false, but the plan would fully rebuild it ({full_rebuild_cause})" - ), - ( - Some(FULL_REFRESH_DISABLED_REBUILD_CODE), - Some( - "set replay_on_change to forward or bounded- on this model to keep its existing history, or remove full_refresh false to allow the rebuild", - ), - ), - )) -} - -/// `check_full_rebuild_allowed` as a function of `module`. -pub(crate) fn check_full_rebuild_allowed_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(check_full_rebuild_allowed, module) -} - -/// `describe_full_rebuild_cause` as a function of `module`. -pub(crate) fn describe_full_rebuild_cause_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(describe_full_rebuild_cause, module) + config_value(model.py(), &model_values(model)?, FULL_REFRESH_KEY) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs index ebfa1985dd..14cb257a33 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs @@ -1,17 +1,14 @@ //! Warehouse-aware time-travel retention and Snowflake table-type planning. +use crate::planner_phases::main::rebuilds::effective_target_config::effective_target_config; use sqlbuild_host::entries::models::AdapterHost; use crate::planner_phases::_helpers::inputs::runtime::runtime_field; -use crate::planner_phases::constants::BIGQUERY_ADAPTER; -use crate::planner_phases::constants::DECREASE_WARNING; -use crate::planner_phases::constants::DOWNGRADE_HISTORY_DAYS; -use crate::planner_phases::constants::PERMANENT_TABLE_TYPE; -use crate::planner_phases::constants::RETENTION_DECREASE; -use crate::planner_phases::constants::RETENTION_INCREASE; -use crate::planner_phases::constants::RETENTION_MATCH; -use crate::planner_phases::constants::SNOWFLAKE_ADAPTER; -use crate::planner_phases::constants::TRANSIENT_TABLE_TYPE; +use crate::planner_phases::constants::{ + BIGQUERY_ADAPTER, DECREASE_WARNING, DOWNGRADE_HISTORY_DAYS, PERMANENT_TABLE_TYPE, + RETENTION_DECREASE, RETENTION_INCREASE, RETENTION_MATCH, SNOWFLAKE_ADAPTER, + TRANSIENT_TABLE_TYPE, +}; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -25,7 +22,6 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; -use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::typed_values::typed_values; use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; @@ -33,7 +29,6 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::name_object: use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::planner_phases::main::construct::construct; use sqlbuild_python_project::planner_phases::main::member::member; -use sqlbuild_python_project::planner_phases::main::native::native; use sqlbuild_python_project::planner_phases::main::native_member::native_member; use sqlbuild_python_project::planner_phases::main::optional_text::optional_text; use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; @@ -47,20 +42,6 @@ use std::collections::{BTreeMap, BTreeSet}; const MATERIALIZED_KEY: &str = "materialized"; -/// The resolved config of the planned target, or defaults without one. -pub(crate) fn effective_target_config<'py>( - runtime: &Bound<'py, PyAny>, -) -> PyResult> { - let py = runtime.py(); - let project_config = runtime_field(runtime, |runtime| &runtime.project_config)?; - let local_config = runtime_field(runtime, |runtime| &runtime.local_config)?; - let target_name = native_field(&native_field(runtime, "project")?, "effective_target_name")?; - if project_config.is_none() || local_config.is_none() || target_name.is_none() { - return native(py, RecordId::TargetConfig, &[]); - } - resolve_target_config(py, &project_config, &local_config, &target_name) -} - /// A deterministic identifier-fitted conversion copy name. #[pyfunction] #[pyo3(signature = (*, target_name, identifier_limit))] @@ -658,24 +639,3 @@ pub(crate) fn plan_retention<'py>( } tuple(py, decided) } - -/// `plan_retention` as a function of `module`. -pub(crate) fn plan_retention_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(plan_retention, module) -} - -/// `plan_table_types` as a function of `module`. -pub(crate) fn plan_table_types_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(plan_table_types, module) -} - -/// `table_type_copy_name` as a function of `module`. -pub(crate) fn table_type_copy_name_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(table_type_copy_name, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/class_field.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/class_field.rs index ba79b54666..15dda60248 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/class_field.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/class_field.rs @@ -2,10 +2,7 @@ use crate::planner_phases::_helpers::inputs::contexts; use pyo3::PyClass; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, Py, PyAny, PyResult}; use pyo3::pyclass::boolean_struct::True; /// One field of a frozen native planner class passed as a Python object. diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/gather_planner_warehouse_state.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/gather_planner_warehouse_state.rs index e78e98b11d..761cf17497 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/gather_planner_warehouse_state.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/gather_planner_warehouse_state.rs @@ -1,9 +1,7 @@ //! Gather planner warehouse state. use crate::planner_phases::_helpers::inputs::warehouse_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Gather planner warehouse state. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_field.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_field.rs index 4be5d2937f..f9f1f658c4 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_field.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_field.rs @@ -2,10 +2,7 @@ use crate::planner_phases::_helpers::inputs::runtime; use crate::planner_phases::models::PlannerRuntime; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, Py, PyAny, PyResult}; /// One field of a planner runtime passed as a Python object. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/warehouse_field.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/warehouse_field.rs index 5b957c13ff..35a46a678b 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/warehouse_field.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/warehouse_field.rs @@ -2,10 +2,7 @@ use crate::planner_phases::_helpers::inputs::runtime; use crate::planner_phases::models::PlannerWarehouseState; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, Py, PyAny, PyResult}; /// One field of a planner warehouse state passed as a Python object. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_deferred_origin_notes.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_deferred_origin_notes.rs index 3db676335c..0bbac7ae9d 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_deferred_origin_notes.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_deferred_origin_notes.rs @@ -1,17 +1,72 @@ //! Note unselected upstreams deferred to pre-migration names, and each lookup in `failures`. -use crate::planner_phases::_helpers::outputs::output_assembly; +use crate::planner_phases::_helpers::outputs::output_assembly::concat; +use crate::planner_phases::main::outputs::unselected_upstream_models::unselected_upstream_models; use crate::planner_phases::models::PlanOutputFields; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; /// Note unselected upstreams deferred to pre-migration names, and each lookup in `failures`. -#[inline] pub fn add_deferred_origin_notes<'py>( - output: PlanOutputFields<'py>, + mut output: PlanOutputFields<'py>, (project, scopes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), (deferral, failures): (&Bound<'py, PyAny>, &[(String, String)]), ) -> PyResult> { - output_assembly::add_deferred_origin_notes(output, (project, scopes), (deferral, failures)) + let py = project.py(); + let locations = native_field(deferral, "deferred_locations")?; + if locations.is_none() { + return Ok(output); + } + let names: Vec = unselected_upstream_models(scopes)?; + let warning = |name: &str, message: &str| -> PyResult> { + native( + py, + RecordId::PlanWarning, + &[ + ("model_name", text(py, name)), + ( + "severity", + enum_member(py, RecordId::WarningSeverity, "WARNING")?, + ), + ("message", text(py, message)), + ], + ) + }; + let mut notes: Vec> = Vec::new(); + for model in native_field(project, "models")?.try_iter()? { + let model = model?; + let name: String = native_field(&model, "name")?.extract()?; + if names.binary_search(&name).is_err() { + continue; + } + let Some(location) = locations.downcast::()?.get_item(&name)? else { + continue; + }; + if native_field(&location, "name")? + .eq(native_field(&native_field(&model, "destination")?, "name")?)? + { + continue; + } + notes.push(warning( + &name, + &format!( + "deferred to pre-migration name {}: the deferred target has not built model '{name}' under its current name yet", + native_field(&location, "qualified_name")? + ), + )?); + } + for (name, message) in failures { + notes.push(warning(name, message)?); + } + if !notes.is_empty() { + let warnings = concat(py, &[output.field("warnings")?, tuple(py, notes)?])?; + output.set("warnings", warnings); + } + Ok(output) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_metadata.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_metadata.rs index eaa8972aa0..c6605b8674 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_metadata.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_metadata.rs @@ -1,17 +1,33 @@ //! Add the direct-mode source freshness, remaining stale models and selection diagnostics metadata. -use crate::planner_phases::_helpers::outputs::output_assembly; +use crate::planner_phases::_helpers::outputs::output_assembly::freshness_metadata; use crate::planner_phases::models::PlanOutputFields; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_phases::main::sorted_names::sorted_names; +use sqlbuild_python_project::planner_phases::main::text::text; /// Add the direct-mode source freshness, remaining stale models and selection diagnostics metadata. -#[inline] pub fn add_plan_metadata<'py>( - output: PlanOutputFields<'py>, + mut output: PlanOutputFields<'py>, source_freshness: &Bound<'py, PyAny>, policies: &Bound<'py, PyAny>, ) -> PyResult> { - output_assembly::add_plan_metadata(output, source_freshness, policies) + let py = source_freshness.py(); + let diagnostics = PyDict::new(py); + diagnostics.set_item("mode", text(py, "direct"))?; + diagnostics.set_item("enabled", native_field(policies, "selection_diagnostics")?)?; + output.metadata.push(( + "direct_source_freshness", + freshness_metadata(source_freshness)?, + )); + output.metadata.push(( + "direct_remaining_stale_model_names", + sorted_names(py, Vec::new())?, + )); + output + .metadata + .push(("selection_diagnostics", diagnostics.into_any())); + Ok(output) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_warnings.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_warnings.rs index 6e42a48d1d..3eb7def0e1 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_warnings.rs @@ -1,11 +1,8 @@ //! Add unknown source freshness warnings and, with selection diagnostics, stale-selection warnings. use crate::planner_phases::_helpers::outputs::output_assembly; -use crate::planner_phases::models::PlanOutputFields; -use crate::planner_phases::models::PlanWarningContext; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_phases::models::{PlanOutputFields, PlanWarningContext}; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Add unknown source freshness warnings and, with selection diagnostics, stale-selection warnings. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_storage_policies.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_storage_policies.rs index 212eae9c22..fa3d16dae6 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_storage_policies.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_storage_policies.rs @@ -1,8 +1,7 @@ //! Add migration fingerprints, rename hints, migration, table-type and retention entries. use crate::planner_phases::_helpers::outputs::output_assembly; -use crate::planner_phases::models::PlanOutputFields; -use crate::planner_phases::models::StoragePolicyContext; +use crate::planner_phases::models::{PlanOutputFields, StoragePolicyContext}; use pyo3::prelude::PyResult; /// Add migration fingerprints, rename hints, migration, table-type and retention entries. diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/base_plan_output.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/base_plan_output.rs index 3db2cd82d9..bae81b84cf 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/base_plan_output.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/base_plan_output.rs @@ -1,12 +1,8 @@ //! The plan output's base fields from the planned entries, relations and source freshness. use crate::planner_phases::_helpers::outputs::output_assembly; -use crate::planner_phases::models::PlanOutputContext; -use crate::planner_phases::models::PlanOutputFields; -use crate::planner_phases::models::PlannedModelEntries; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_phases::models::{PlanOutputContext, PlanOutputFields, PlannedModelEntries}; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// The plan output's base fields from the planned entries, relations and source freshness. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/build_planner_identity_context.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/build_planner_identity_context.rs index dd98583df5..a1cdf347d2 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/build_planner_identity_context.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/build_planner_identity_context.rs @@ -1,20 +1,39 @@ //! Build planner identity context. -use crate::planner_phases::_helpers::outputs::identities; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_phases::_helpers::inputs::contexts::class_field; +use crate::planner_phases::_helpers::outputs::identities::version_identities; +use crate::planner_phases::models::PlannerIdentityContext; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; /// Build planner identity context. -#[inline] pub fn build_planner_identity_context<'py>( project: &Bound<'py, PyAny>, (scopes, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), include_stale_warning_identities: bool, ) -> PyResult> { - identities::build_planner_identity_context( + let recorded_fingerprints: Bound<'py, PyAny> = native_field(snapshot, "fingerprints")?; + let identities = version_identities( project, - (scopes, snapshot), - include_stale_warning_identities, - ) + class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, + &recorded_fingerprints, + )?; + let stale = if include_stale_warning_identities { + version_identities( + project, + class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)?, + &recorded_fingerprints, + )? + } else { + identities.clone() + }; + Ok(Bound::new( + project.py(), + PlannerIdentityContext { + version_identities: identities.unbind(), + stale_warning_identities: stale.unbind(), + }, + )? + .into_any()) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/check_selected_scope_buildability.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/check_selected_scope_buildability.rs index db8be8e2b6..8090e77bf5 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/check_selected_scope_buildability.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/check_selected_scope_buildability.rs @@ -1,9 +1,7 @@ //! Check selected scope buildability. use crate::planner_phases::_helpers::outputs::buildability; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Check selected scope buildability. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes.rs index f63cf08d39..5a1378b311 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes.rs @@ -1,9 +1,7 @@ //! Changes of stale-warning models whose identities differ from the execution identities. use crate::planner_phases::_helpers::outputs::identities; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Changes of stale-warning models whose identities differ from the execution identities. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes_of_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes_of_function.rs index a7bea34508..be2b04d3fd 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes_of_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes_of_function.rs @@ -1,11 +1,10 @@ //! `detect_stale_warning_changes_of` as a function of `module`. -use crate::planner_phases::_helpers::outputs::identities; +use crate::planner_phases::_helpers::outputs::identities::detect_stale_warning_changes_of; /// `detect_stale_warning_changes_of` as a function of `module`. -#[inline] pub fn detect_stale_warning_changes_of_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - identities::detect_stale_warning_changes_of_function(module) + pyo3::wrap_pyfunction!(detect_stale_warning_changes_of, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/planner_identity_work_info_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/planner_identity_work_info_function.rs index 052eecee81..51d1b092e1 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/planner_identity_work_info_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/planner_identity_work_info_function.rs @@ -1,11 +1,10 @@ //! `planner_identity_work_info` as a function of `module`. -use crate::planner_phases::_helpers::outputs::identities; +use crate::planner_phases::_helpers::outputs::identities::planner_identity_work_info; /// `planner_identity_work_info` as a function of `module`. -#[inline] pub fn planner_identity_work_info_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - identities::planner_identity_work_info_function(module) + pyo3::wrap_pyfunction!(planner_identity_work_info, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/unselected_upstream_models.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/unselected_upstream_models.rs index 3c38fd5cda..0a58ec1c82 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/unselected_upstream_models.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/unselected_upstream_models.rs @@ -1,12 +1,26 @@ //! The unselected models the selected scope reads, sorted. -use crate::planner_phases::_helpers::outputs::output_assembly; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_phases::_helpers::inputs::contexts::class_field; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_analysis::planner_scope::constants::MODEL; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; /// The unselected models the selected scope reads, sorted. -#[inline] pub fn unselected_upstream_models(scopes: &Bound<'_, PyAny>) -> PyResult> { - output_assembly::unselected_upstream_models(scopes) + let selected = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; + let selected = sqlbuild_python_project::planner_scope::main::scope_data::scope_data(&selected)?; + let mut names: Vec = Vec::new(); + for (key, deps) in &selected.upstream { + if !selected.selected.contains(key) { + continue; + } + for dep in deps { + if dep.0 == MODEL && !selected.selected.contains(dep) { + names.push(dep.1.clone()); + } + } + } + names.sort(); + names.dedup(); + Ok(names) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/build_planner_entry_results.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/build_planner_entry_results.rs index 28a4974f80..f13fa09f2f 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/build_planner_entry_results.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/build_planner_entry_results.rs @@ -1,11 +1,8 @@ //! Model plan entries for the pruned execution scope, with the freshness and override inputs they need. use crate::planner_phases::_helpers::rebuilds::execution_changes; -use crate::planner_phases::models::ChangeReconciliation; -use crate::planner_phases::models::ModelEntryResults; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_phases::models::{ChangeReconciliation, ModelEntryResults}; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Model plan entries for the pruned execution scope, with the freshness and override inputs they need. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed.rs index 0148cb6de6..c032730e9e 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed.rs @@ -1,16 +1,41 @@ //! Check full rebuild allowed. -use crate::planner_phases::_helpers::rebuilds::full_refresh; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_phases::_helpers::rebuilds::full_refresh::configured_full_refresh; +use crate::planner_phases::constants::FULL_REFRESH_DISABLED_REBUILD_CODE; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyBool; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; -/// Check full rebuild allowed. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, model, change_result, full_rebuild_cause))] pub fn check_full_rebuild_allowed( model: &Bound<'_, PyAny>, change_result: &Bound<'_, PyAny>, full_rebuild_cause: &str, ) -> PyResult<()> { - full_refresh::check_full_rebuild_allowed(model, change_result, full_rebuild_cause) + let py = model.py(); + if !configured_full_refresh(model)?.is(PyBool::new(py, false)) { + return Ok(()); + } + if native_field(change_result, "change_kind")?.eq("first_run")? + || native_field(change_result, "query_changed")?.is_truthy()? + { + return Ok(()); + } + let name: String = model_name(model)?; + Err(planner_error( + py, + &format!( + "model '{name}' sets full_refresh false, but the plan would fully rebuild it ({full_rebuild_cause})" + ), + ( + Some(FULL_REFRESH_DISABLED_REBUILD_CODE), + Some( + "set replay_on_change to forward or bounded- on this model to keep its existing history, or remove full_refresh false to allow the rebuild", + ), + ), + )) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed_function.rs index e3ce35309f..f1834b4b6f 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed_function.rs @@ -1,11 +1,10 @@ //! `check_full_rebuild_allowed` as a function of `module`. -use crate::planner_phases::_helpers::rebuilds::full_refresh; +use crate::planner_phases::main::rebuilds::check_full_rebuild_allowed::check_full_rebuild_allowed; /// `check_full_rebuild_allowed` as a function of `module`. -#[inline] pub fn check_full_rebuild_allowed_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - full_refresh::check_full_rebuild_allowed_function(module) + pyo3::wrap_pyfunction!(check_full_rebuild_allowed, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/defer_to_migration_origins.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/defer_to_migration_origins.rs index bc0aa22705..b119e3161f 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/defer_to_migration_origins.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/defer_to_migration_origins.rs @@ -1,9 +1,7 @@ //! Redirect unselected `models` unbuilt in the deferred target to their `migrate_from` relation. use crate::planner_phases::_helpers::rebuilds::deferred_origins; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; /// Redirect unselected `models` unbuilt in the deferred target to their `migrate_from` relation. diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause.rs index ec147f0570..bd8115348a 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause.rs @@ -1,15 +1,25 @@ //! Describe full rebuild cause. -use crate::planner_phases::_helpers::rebuilds::full_refresh; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_analysis::planner_warehouse::main::full_rebuild_cause::full_rebuild_cause; +use sqlbuild_analysis::planner_warehouse::models::FullRebuildFacts; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_phases::main::strings::strings; -/// Describe full rebuild cause. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, change_result, full_refresh))] pub fn describe_full_rebuild_cause( change_result: &Bound<'_, PyAny>, full_refresh: bool, ) -> PyResult { - full_refresh::describe_full_rebuild_cause(change_result, full_refresh) + Ok(full_rebuild_cause(&FullRebuildFacts { + full_refresh, + change_kind: native_field(change_result, "change_kind")? + .str()? + .extract::()?, + query_changed: native_field(change_result, "query_changed")?.is_truthy()?, + changed_functions: strings(&native_field(change_result, "changed_functions")?)?, + has_schema_findings: native_field(change_result, "schema_findings")?.is_truthy()?, + })) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause_function.rs index 5a5a37bf68..61c65cebe4 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause_function.rs @@ -1,11 +1,10 @@ //! `describe_full_rebuild_cause` as a function of `module`. -use crate::planner_phases::_helpers::rebuilds::full_refresh; +use crate::planner_phases::main::rebuilds::describe_full_rebuild_cause::describe_full_rebuild_cause; /// `describe_full_rebuild_cause` as a function of `module`. -#[inline] pub fn describe_full_rebuild_cause_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - full_refresh::describe_full_rebuild_cause_function(module) + pyo3::wrap_pyfunction!(describe_full_rebuild_cause, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effective_target_config.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effective_target_config.rs index 0745520645..359c74b55a 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effective_target_config.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effective_target_config.rs @@ -1,12 +1,20 @@ //! The resolved config of the planned target, or defaults without one. -use crate::planner_phases::_helpers::rebuilds::retention; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_phases::_helpers::inputs::runtime::runtime_field; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_project::planner_phases::main::native::native; /// The resolved config of the planned target, or defaults without one. -#[inline] pub fn effective_target_config<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { - retention::effective_target_config(runtime) + let py = runtime.py(); + let project_config = runtime_field(runtime, |runtime| &runtime.project_config)?; + let local_config = runtime_field(runtime, |runtime| &runtime.local_config)?; + let target_name = native_field(&native_field(runtime, "project")?, "effective_target_name")?; + if project_config.is_none() || local_config.is_none() || target_name.is_none() { + return native(py, RecordId::TargetConfig, &[]); + } + resolve_target_config(py, &project_config, &local_config, &target_name) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs index 93ea4b0e66..2e93d90235 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs @@ -1,15 +1,22 @@ //! Names of models effectively full refreshed for this invocation. -use crate::planner_phases::_helpers::rebuilds::full_refresh; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_phases::main::rebuilds::resolve_model_full_refresh::resolve_model_full_refresh; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; /// Names of models effectively full refreshed for this invocation. -#[inline] pub fn effectively_full_refreshed_model_names<'py>( project: &Bound<'py, PyAny>, cli_full_refresh: bool, ) -> PyResult> { - full_refresh::effectively_full_refreshed_model_names(project, cli_full_refresh) + let mut names: Vec<_> = Vec::new(); + for model in compiled_models(project)?.try_iter()? { + let model = model?; + if resolve_model_full_refresh(&model, cli_full_refresh)? { + names.push(name_object(&model)?); + } + } + frozenset(project.py(), &names) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_retention_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_retention_function.rs index d2bcbeb000..a80cee656b 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_retention_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_retention_function.rs @@ -1,11 +1,10 @@ //! `plan_retention` as a function of `module`. -use crate::planner_phases::_helpers::rebuilds::retention; +use crate::planner_phases::_helpers::rebuilds::retention::plan_retention; /// `plan_retention` as a function of `module`. -#[inline] pub fn plan_retention_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - retention::plan_retention_function(module) + pyo3::wrap_pyfunction!(plan_retention, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_table_types_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_table_types_function.rs index d239a14705..308dc0630f 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_table_types_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_table_types_function.rs @@ -1,11 +1,10 @@ //! `plan_table_types` as a function of `module`. -use crate::planner_phases::_helpers::rebuilds::retention; +use crate::planner_phases::_helpers::rebuilds::retention::plan_table_types; /// `plan_table_types` as a function of `module`. -#[inline] pub fn plan_table_types_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - retention::plan_table_types_function(module) + pyo3::wrap_pyfunction!(plan_table_types, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/reconcile_execution_changes.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/reconcile_execution_changes.rs index 95b3dee371..1a5e191553 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/reconcile_execution_changes.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/reconcile_execution_changes.rs @@ -1,23 +1,41 @@ //! The execution scope's changes with honest write hashes, and the actions merged from them. -use crate::planner_phases::_helpers::rebuilds::execution_changes; +use crate::planner_identity::main::merge_recomputed_model_changes::merge_recomputed_model_changes; +use crate::planner_identity::main::with_honest_model_write_hashes::with_honest_model_write_hashes; +use crate::planner_phases::_helpers::inputs::contexts::class_field; use crate::planner_phases::models::ChangeReconciliation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_phases::models::PlannerIdentityContext; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::boundary::main::keywords::keywords; /// The execution scope's changes with honest write hashes, and the actions merged from them. -#[inline] pub fn reconcile_execution_changes<'py>( (execution_scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), identities: &Bound<'py, PyAny>, resolved_actions: &Bound<'py, PyAny>, changes: &Bound<'py, PyAny>, ) -> PyResult> { - execution_changes::reconcile_execution_changes( - (execution_scope, snapshot), - identities, - resolved_actions, - changes, - ) + let py = execution_scope.py(); + let honest = with_honest_model_write_hashes( + py, + Some(keywords( + py, + &[ + ("scope", execution_scope.clone()), + ("snapshot", snapshot.clone()), + ("changes", changes.clone()), + ( + "version_identities", + class_field(identities, |i: &PlannerIdentityContext| { + &i.version_identities + })?, + ), + ], + )?), + )?; + let resolved = merge_recomputed_model_changes(resolved_actions, &honest)?; + Ok(ChangeReconciliation { + changes: honest, + resolved_actions: resolved, + }) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/resolve_model_full_refresh.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/resolve_model_full_refresh.rs index 0b197a08b7..0e073d4dff 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/resolve_model_full_refresh.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/resolve_model_full_refresh.rs @@ -1,15 +1,17 @@ //! dbt-compatible nullable model full-refresh semantics. -use crate::planner_phases::_helpers::rebuilds::full_refresh; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_phases::_helpers::rebuilds::full_refresh::configured_full_refresh; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyBool; /// dbt-compatible nullable model full-refresh semantics. -#[inline] pub fn resolve_model_full_refresh( model: &Bound<'_, PyAny>, cli_full_refresh: bool, ) -> PyResult { - full_refresh::resolve_model_full_refresh(model, cli_full_refresh) + let configured = configured_full_refresh(model)?; + if configured.is_instance_of::() { + return configured.extract(); + } + Ok(cli_full_refresh) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/table_type_copy_name_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/table_type_copy_name_function.rs index f58f400825..56017b44d1 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/table_type_copy_name_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/table_type_copy_name_function.rs @@ -1,11 +1,10 @@ //! `table_type_copy_name` as a function of `module`. -use crate::planner_phases::_helpers::rebuilds::retention; +use crate::planner_phases::_helpers::rebuilds::retention::table_type_copy_name; /// `table_type_copy_name` as a function of `module`. -#[inline] pub fn table_type_copy_name_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - retention::table_type_copy_name_function(module) + pyo3::wrap_pyfunction!(table_type_copy_name, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/models.rs b/crates/sqlbuild-python-planner/src/planner_phases/models.rs index ec83416021..a7a3da7fd2 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/models.rs @@ -1,8 +1,6 @@ //! Public records of planner phases. -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; +use pyo3::prelude::{Bound, Py, PyAny}; use pyo3::pyclass; use std::collections::BTreeMap; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_bounds.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_bounds.rs index 489e89dc70..d42cd6fc87 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_bounds.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_bounds.rs @@ -4,9 +4,9 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::planner_resolution::_helpers::references::values::string; use crate::planner_resolution::_helpers::safety::maximum_safety::apply_maximum_start_policy; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyDict; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_cursor::algebra::main::inclusive_to_exclusive::inclusive_to_exclusive; use sqlbuild_cursor::algebra::models::{DurationSpan, Grain, Scalar}; use sqlbuild_cursor::planner::main::raw_bounds::raw_bounds; @@ -299,8 +299,3 @@ pub(crate) fn compute_cursor_bounds<'py>( } apply_maximum_start_policy(py, Some(keywords(py, &policy_args)?)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(compute_cursor_bounds, module)?)?; - module.add_function(wrap_pyfunction!(apply_typed_cursor_replay_policy, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs index a5ba16028f..f518f20798 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs @@ -6,8 +6,9 @@ use crate::plan_output::main::resolve_bounded_cursor_override::resolve_bounded_c use crate::planner_entries::models::ModelPlanContext; use crate::planner_phases::main::inputs::class_field::class_field; use crate::planner_resolution::_helpers::cursor::cursor_bounds::compute_cursor_bounds; -use crate::planner_resolution::_helpers::cursor::cursor_policies::future_cursor_config; -use crate::planner_resolution::_helpers::cursor::cursor_policies::start_cursor_config; +use crate::planner_resolution::_helpers::cursor::cursor_policies::{ + future_cursor_config, start_cursor_config, +}; use crate::planner_resolution::_helpers::cursor::cursor_windows::cursor_inputs_without_rows; use crate::planner_resolution::_helpers::cursor::cursor_windows::empty_cursor_window; use crate::planner_resolution::_helpers::cursor::cursor_windows::empty_input_rebuild_refusal; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs index 67f36b7065..a6b2ad573a 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs @@ -1,12 +1,9 @@ //! Selected-model inspection and untyped override conversion. -use crate::planner_resolution::_helpers::references::values::method; -use crate::planner_resolution::_helpers::references::values::string; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use crate::planner_resolution::_helpers::references::values::{method, string}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cursor::planner::main::override_kind::override_kind; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -29,7 +26,7 @@ const TIMESTAMP_CURSOR: &str = "timestamp"; #[pyfunction(name = "type_cursor_overrides")] #[pyo3(signature = (**kwargs))] -fn type_cursor_overrides_of<'py>( +pub(crate) fn type_cursor_overrides_of<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -96,7 +93,3 @@ pub(crate) fn type_cursor_overrides<'py>( let kind = enum_from_value(py, RecordId::CursorType, kind)?; method(overrides, "typed_as", &[("cursor_type", kind)]) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(type_cursor_overrides_of, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_policies.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_policies.rs index 50a8bcd9a5..978a01dd64 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_policies.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_policies.rs @@ -82,11 +82,3 @@ pub(crate) fn start_cursor_config<'py>( ) -> PyResult> { resolve_policy(model, project_config, &START_POLICY) } - -/// Resolve automatic-start policy with model config taking precedence. -pub(crate) fn resolve_start_cursor_config<'py>( - model: &Bound<'py, PyAny>, - project_config: &Bound<'py, PyAny>, -) -> PyResult> { - start_cursor_config(model, project_config) -} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs index 56a49d21cb..31a7c01bb8 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs @@ -1,28 +1,25 @@ //! Plan-entry cursor windows: overrides, planner-owned bounds and microbatch ranges. +use crate::planner_resolution::main::cursor::config_str::config_str; +use crate::planner_resolution::main::cursor::model_cursor_overrides::model_cursor_overrides; +use crate::planner_resolution::main::cursor::plan_cursor_bounds::plan_cursor_bounds; +use crate::planner_resolution::main::cursor::resolve_effective_timestamp_grain::resolve_effective_timestamp_grain; use crate::planner_resolution::models::CursorWindowInputs; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use crate::plan_output::main::resolve_bounded_cursor_override::resolve_bounded_cursor_override; -use crate::planner_entries::models::ModelPlanContext; -use crate::planner_phases::main::inputs::class_field::class_field; use crate::planner_resolution::_helpers::cursor::cursor_bounds::compute_cursor_bounds; use crate::planner_resolution::_helpers::cursor::cursor_bounds::duration; -use crate::planner_resolution::_helpers::cursor::cursor_policies::future_cursor_config; -use crate::planner_resolution::_helpers::cursor::cursor_policies::start_cursor_config; -use crate::planner_resolution::_helpers::cursor::cursor_safety::replace; -use crate::planner_resolution::_helpers::cursor::cursor_safety::utc_clock; -use crate::planner_resolution::_helpers::cursor::cursor_windows::cursor_inputs_without_rows; +use crate::planner_resolution::_helpers::cursor::cursor_safety::{replace, utc_clock}; use crate::planner_resolution::_helpers::cursor::cursor_windows::normalize_cursor_snapshot_grain; -use crate::planner_resolution::_helpers::cursor::cursor_windows::resolve_effective_timestamp_grain; use crate::planner_resolution::_helpers::cursor::cursor_windows::without_destination_cursor; use crate::planner_resolution::_helpers::references::values::string; use crate::planner_resolution::_helpers::safety::future_safety::resolve_future_cursor_safety; use crate::planner_strategy::main::model_entries::resolve_microbatch_lookback::resolve_microbatch_lookback; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, PyErr, pyfunction}; use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::models::{Scalar, ScalarValue, Timestamp, ZoneOffset}; use sqlbuild_cursor::planner::main::cursor_end_clamp::cursor_end_clamp; @@ -43,19 +40,18 @@ use sqlbuild_python_front::cursor::main::values::local_date_time::local_date_tim use sqlbuild_python_front::cursor::main::values::replaced::replaced; use sqlbuild_python_front::cursor::models::CursorSession; use sqlbuild_python_front::model_config::main::config::config_cursor_bound::config_cursor_bound as typed_cursor_bound; -use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::planner_phases::main::object::object; use std::cmp::Ordering; -const INCREMENTAL: &str = "incremental"; -const MICROBATCH: &str = "microbatch"; -const ROLLING_WINDOW: &str = "rolling_window"; -const TIMESTAMP: &str = "timestamp"; -const INTEGER: &str = "integer"; +pub(crate) const INCREMENTAL: &str = "incremental"; +pub(crate) const MICROBATCH: &str = "microbatch"; +pub(crate) const ROLLING_WINDOW: &str = "rolling_window"; +pub(crate) const TIMESTAMP: &str = "timestamp"; +pub(crate) const INTEGER: &str = "integer"; const BOUNDED_BACKFILL: &str = "bounded"; -const FULL_BACKFILL: &str = "full"; +pub(crate) const FULL_BACKFILL: &str = "full"; const WINDOW_REQUIRED: [&str; 7] = [ "model", "snapshot", @@ -73,13 +69,6 @@ const WINDOW_OPTIONAL: [&str; 5] = [ "cursor_input_relations", ]; -/// One string-valued model configuration entry. -pub(crate) fn config_str<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { - let py: Python<'py> = model.py(); - Ok(config_text_object(py, &model_values(model)?, key)? - .unwrap_or_else(|| py.None().into_bound(py))) -} - /// One cursor-bound model configuration entry as its token text. pub(crate) fn config_bound<'py>( model: &Bound<'py, PyAny>, @@ -108,7 +97,7 @@ fn order_scalar<'py>( session.scalar(&parse_value(token, &string(py, kind))?) } -fn order<'py>( +pub(crate) fn order<'py>( session: &CursorSession<'py>, tokens: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), cursor_type: &Bound<'py, PyAny>, @@ -118,7 +107,10 @@ fn order<'py>( session.checked(compare(&left, &right, session)) } -fn token<'py>(py: Python<'py>, bound: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn token<'py>( + py: Python<'py>, + bound: &Bound<'py, PyAny>, +) -> PyResult> { Ok(PyString::new(py, &sentinel_to_token(bound)?).into_any()) } @@ -127,7 +119,7 @@ fn sentinel<'py>(py: Python<'py>, name: &str) -> PyResult> { } /// `bounds` clamped so a concrete window never reaches the exclusive configured `cursor_end`. -fn clamp_cursor_end<'py>( +pub(crate) fn clamp_cursor_end<'py>( bounds: Bound<'py, PyAny>, model: &Bound<'py, PyAny>, ) -> PyResult> { @@ -270,7 +262,7 @@ fn apply_future_safety<'py>( ) } -fn bounded_override<'py>( +pub(crate) fn bounded_override<'py>( model: &Bound<'py, PyAny>, inputs: &CursorWindowInputs<'py>, ) -> PyResult> { @@ -293,14 +285,14 @@ fn backfill_duration<'py>(backfill: &Bound<'py, PyAny>) -> PyResult { - cursor_snapshot: Bound<'py, PyAny>, - typing: (Bound<'py, PyAny>, Bound<'py, PyAny>), - lookback: Bound<'py, PyAny>, - is_microbatch: bool, +pub(crate) struct BoundsRequest<'py> { + pub(crate) cursor_snapshot: Bound<'py, PyAny>, + pub(crate) typing: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) lookback: Bound<'py, PyAny>, + pub(crate) is_microbatch: bool, } -fn snapshot_bounds<'py>( +pub(crate) fn snapshot_bounds<'py>( model: &Bound<'py, PyAny>, inputs: &CursorWindowInputs<'py>, request: BoundsRequest<'py>, @@ -346,55 +338,17 @@ fn snapshot_bounds<'py>( clamp_cursor_end(bounds, model) } -fn model_cursor_snapshot<'py>( +pub(crate) fn model_cursor_snapshot<'py>( model: &Bound<'py, PyAny>, snapshot: &Bound<'py, PyAny>, ) -> PyResult> { native_field(snapshot, "cursor_snapshots")?.call_method1("get", (name_object(model)?,)) } -fn without_destination<'py>(cursor_snapshot: Bound<'py, PyAny>) -> PyResult> { - without_destination_cursor(&cursor_snapshot) -} - -/// Compute cursor bounds for inclusion on the plan entry. -pub(crate) fn plan_cursor_bounds<'py>( - model: &Bound<'py, PyAny>, - inputs: &CursorWindowInputs<'py>, +pub(crate) fn without_destination<'py>( + cursor_snapshot: Bound<'py, PyAny>, ) -> PyResult> { - let py = model.py(); - if !config_str(model, "materialized")?.eq(INCREMENTAL)? - || config_str(model, "cursor")?.is_none() - { - return Ok(py.None().into_bound(py)); - } - let bounded = bounded_override(model, inputs)?; - if !bounded.is_none() { - return clamp_cursor_end(bounded, model); - } - if inputs.runtime_owned || inputs.full_refresh { - return Ok(py.None().into_bound(py)); - } - let mut cursor_snapshot = model_cursor_snapshot(model, &inputs.snapshot)?; - if cursor_snapshot.is_none() { - return Ok(cursor_snapshot); - } - if native_field(&inputs.backfill, "action")?.eq(FULL_BACKFILL)? { - cursor_snapshot = without_destination(cursor_snapshot)?; - } - snapshot_bounds( - model, - inputs, - BoundsRequest { - cursor_snapshot, - typing: ( - config_str(model, "cursor_type")?, - config_str(model, "cursor_grain")?, - ), - lookback: config_str(model, "lookback")?, - is_microbatch: config_str(model, "incremental_mode")?.eq(MICROBATCH)?, - }, - ) + without_destination_cursor(&cursor_snapshot) } /// Compute the real overall cursor range for microbatch batch splitting. @@ -465,119 +419,10 @@ pub(crate) fn microbatch_range<'py>( ) } -/// Empty inputs that leave a planner-owned window empty, and whether others have rows. -pub(crate) fn empty_cursor_inputs<'py>( - model: &Bound<'py, PyAny>, - inputs: &CursorWindowInputs<'py>, -) -> PyResult<(Bound<'py, PyAny>, bool)> { - let py = model.py(); - let nothing = - || -> PyResult<(Bound<'py, PyAny>, bool)> { Ok((PyTuple::empty(py).into_any(), false)) }; - if inputs.full_refresh - || inputs.runtime_owned - || !config_str(model, "materialized")?.eq(INCREMENTAL)? - || config_str(model, "cursor")?.is_none() - || config_str(model, "microbatch_strategy")?.eq(ROLLING_WINDOW)? - || (!inputs.overrides.0.is_none() && !inputs.overrides.1.is_none()) - { - return nothing(); - } - let cursor_snapshot = model_cursor_snapshot(model, &inputs.snapshot)?; - if cursor_snapshot.is_none() { - return nothing(); - } - let decision = cursor_inputs_without_rows(&cursor_snapshot)?; - Ok(( - native_field(&decision, "input_names")?, - native_field(&decision, "waiting_on_empty_inputs")?.is_truthy()?, - )) -} - -/// The future-end and automatic-start policies effective for one model. -pub(crate) fn cursor_safety_configs<'py>( - model: &Bound<'py, PyAny>, - context: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - Ok(( - future_cursor_config( - model, - &class_field(context, |c: &ModelPlanContext| &c.future_cursor_config)?, - )?, - start_cursor_config( - model, - &class_field(context, |c: &ModelPlanContext| &c.start_cursor_config)?, - )?, - )) -} - -fn compile_input_error(py: Python<'_>, message: String) -> PyErr { +pub(crate) fn compile_input_error(py: Python<'_>, message: String) -> PyErr { new_plain_error(py, ErrorId::CompileInput, &message) } -/// Resolve typed cursor overrides to generic start/end for one model. -pub(crate) fn model_cursor_overrides<'py>( - model: &Bound<'py, PyAny>, - cursor_overrides: &Bound<'py, PyAny>, - (start, end): (Bound<'py, PyAny>, Bound<'py, PyAny>), -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - let py = model.py(); - let cursor_type = config_str(model, "cursor_type")?; - let (mut resolved_start, mut resolved_end) = (start, end); - let typed_fields = if cursor_overrides.is_none() { - None - } else if cursor_type.eq(TIMESTAMP)? { - Some(("start_ts", "end_ts")) - } else if cursor_type.eq(INTEGER)? { - Some(("start_int", "end_int")) - } else { - None - }; - if let Some((start_field, end_field)) = typed_fields { - let typed_start = native_field(cursor_overrides, start_field)?; - if !typed_start.is_none() { - resolved_start = token(py, &typed_start)?; - } - let typed_end = native_field(cursor_overrides, end_field)?; - if !typed_end.is_none() { - resolved_end = token(py, &typed_end)?; - } - } - let session = CursorSession::new(py)?; - let name = name_object(model)?.str()?; - let cursor_start = config_bound(model, "cursor_start")?; - let cursor_end = config_bound(model, "cursor_end")?; - if !resolved_start.is_none() - && !cursor_start.is_none() - && order(&session, (&resolved_start, &cursor_start), &cursor_type)? == Ordering::Less - { - return Err(compile_input_error( - py, - format!( - "model '{name}': explicit start {} is before cursor_start {}", - resolved_start.str()?, - cursor_start.str()? - ), - )); - } - if !cursor_end.is_none() { - for (value, label) in [(&resolved_start, "start"), (&resolved_end, "end")] { - if !value.is_none() - && order(&session, (value, &cursor_end), &cursor_type)? != Ordering::Less - { - return Err(compile_input_error( - py, - format!( - "model '{name}': explicit {label} {} is at or beyond exclusive cursor_end {}", - value.str()?, - cursor_end.str()? - ), - )); - } - } - } - Ok((resolved_start, resolved_end)) -} - fn window_inputs<'py>(args: &KeywordArguments<'py>) -> PyResult> { Ok(CursorWindowInputs { snapshot: args.value("snapshot")?, @@ -605,7 +450,7 @@ fn window_inputs<'py>(args: &KeywordArguments<'py>) -> PyResult( +pub(crate) fn compute_plan_cursor_bounds<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -621,7 +466,7 @@ fn compute_plan_cursor_bounds<'py>( /// Compute the real overall cursor range for microbatch batch splitting. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn compute_microbatch_range<'py>( +pub(crate) fn compute_microbatch_range<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -637,7 +482,7 @@ fn compute_microbatch_range<'py>( /// Resolve typed cursor overrides to generic start/end for one model. #[pyfunction] #[pyo3(signature = (*, model, cursor_overrides, start_cursor_override, end_cursor_override))] -fn resolve_cursor_overrides<'py>( +pub(crate) fn resolve_cursor_overrides<'py>( model: &Bound<'py, PyAny>, cursor_overrides: &Bound<'py, PyAny>, start_cursor_override: Bound<'py, PyAny>, @@ -649,9 +494,3 @@ fn resolve_cursor_overrides<'py>( (start_cursor_override, end_cursor_override), ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(compute_plan_cursor_bounds, module)?)?; - module.add_function(wrap_pyfunction!(compute_microbatch_range, module)?)?; - module.add_function(wrap_pyfunction!(resolve_cursor_overrides, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_warnings.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_warnings.rs index 6dfd368657..6d1b17c27e 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_warnings.rs @@ -1,17 +1,13 @@ //! Prominent warnings for applied future-end and automatic-start cursor safety. -use crate::planner_resolution::_helpers::references::values::string; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyString; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; -const FUTURE_CAPPED_CODE: &str = "FUTURE_CURSOR_CAPPED"; -const START_CAPPED_CODE: &str = "AUTOMATIC_START_CAPPED"; +pub(crate) const FUTURE_CAPPED_CODE: &str = "FUTURE_CURSOR_CAPPED"; +pub(crate) const START_CAPPED_CODE: &str = "AUTOMATIC_START_CAPPED"; fn rendered(evidence: &Bound<'_, PyAny>, field: &str) -> PyResult { render_value(&native_field(evidence, field)?) @@ -65,53 +61,16 @@ pub(crate) fn maximum_start_message(bounds: &Bound<'_, PyAny>) -> PyResult( - warnings: &Bound<'py, PyAny>, - model_name: &Bound<'py, PyAny>, - bounds: &Bound<'py, PyAny>, -) -> PyResult> { - let py = warnings.py(); - let mut all: Vec> = warnings - .try_iter()? - .collect::>>>()?; - for (message, code) in [ - (future_cap_message(bounds)?, FUTURE_CAPPED_CODE), - (maximum_start_message(bounds)?, START_CAPPED_CODE), - ] { - let Some(message) = message else { continue }; - all.push(new_record_with( - py, - RecordId::PlanWarning, - &[ - ("model_name", model_name.clone()), - ( - "severity", - enum_member(py, RecordId::WarningSeverity, "WARNING")?, - ), - ("code", string(py, code)), - ("message", string(py, &message)), - ], - )?); - } - Ok(PyTuple::new(py, all)?.into_any()) -} - /// Render a prominent warning for applied future cursor policy. #[pyfunction] #[pyo3(signature = (bounds, /))] -fn future_cursor_cap_warning(bounds: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn future_cursor_cap_warning(bounds: &Bound<'_, PyAny>) -> PyResult> { future_cap_message(bounds) } /// Render a warning that distinguishes start recovery from future-end capping. #[pyfunction] #[pyo3(signature = (bounds, /))] -fn maximum_start_cap_warning(bounds: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn maximum_start_cap_warning(bounds: &Bound<'_, PyAny>) -> PyResult> { maximum_start_message(bounds) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(future_cursor_cap_warning, module)?)?; - module.add_function(wrap_pyfunction!(maximum_start_cap_warning, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_windows.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_windows.rs index c8bf37def0..779153ef5d 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_windows.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_windows.rs @@ -1,9 +1,9 @@ //! Empty-window decisions and snapshot grain conversion. use crate::planner_resolution::_helpers::references::values::string; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; use sqlbuild_cursor::planner::main::empty_inputs::empty_inputs; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -19,7 +19,7 @@ use sqlbuild_python_front::cursor::models::CursorSession; #[pyfunction] #[pyo3(signature = (*, empty_input_names, waits_for_every_input, inputs_with_rows, has_window_start))] -fn decide_empty_cursor_inputs<'py>( +pub(crate) fn decide_empty_cursor_inputs<'py>( empty_input_names: &Bound<'py, PyAny>, waits_for_every_input: bool, inputs_with_rows: bool, @@ -105,7 +105,7 @@ pub(crate) fn empty_cursor_window<'py>( #[pyfunction] #[pyo3(signature = (*, cursor_type))] -fn cursor_window_override_flags(cursor_type: Option<&str>) -> String { +pub(crate) fn cursor_window_override_flags(cursor_type: Option<&str>) -> String { let suffix = if cursor_type == Some("integer") { "int" } else { @@ -178,41 +178,6 @@ pub(crate) fn without_destination_cursor<'py>( replaced_keywords(cursor_snapshot, &kwargs) } -#[pyfunction] -#[pyo3(signature = (*, cursor_type, downstream_grain, cursor_input_grains, microbatch_strategy=None))] -pub(crate) fn resolve_effective_timestamp_grain<'py>( - cursor_type: &Bound<'py, PyAny>, - downstream_grain: &Bound<'py, PyAny>, - cursor_input_grains: &Bound<'py, PyAny>, - microbatch_strategy: Option<&str>, -) -> PyResult> { - let py = cursor_type.py(); - if !cursor_type.eq("timestamp")? { - return Ok(py.None().into_bound(py)); - } - let mut effective = if downstream_grain.is_truthy()? { - downstream_grain.clone() - } else { - string(py, "second") - }; - if microbatch_strategy == Some("watermark") { - return Ok(effective); - } - let session = CursorSession::new(py)?; - for value in cursor_input_grains.try_iter()? { - let value = value?; - let candidate = if value.is_truthy()? { - value - } else { - string(py, "second") - }; - if session.grain(&candidate)? > session.grain(&effective)? { - effective = candidate; - } - } - Ok(effective) -} - #[pyfunction] #[pyo3(signature = (*, cursor_snapshot, cursor_type, effective_grain))] pub(crate) fn normalize_cursor_snapshot_grain<'py>( @@ -261,12 +226,3 @@ pub(crate) fn normalize_cursor_snapshot_grain<'py>( kwargs.set_item("upstream_end_inputs", PyTuple::new(py, ends)?)?; replaced_keywords(cursor_snapshot, &kwargs) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(decide_empty_cursor_inputs, module)?)?; - module.add_function(wrap_pyfunction!(empty_cursor_window, module)?)?; - module.add_function(wrap_pyfunction!(cursor_window_override_flags, module)?)?; - module.add_function(wrap_pyfunction!(empty_input_rebuild_refusal, module)?)?; - module.add_function(wrap_pyfunction!(resolve_effective_timestamp_grain, module)?)?; - module.add_function(wrap_pyfunction!(normalize_cursor_snapshot_grain, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs index f913221385..5ad8ffd24d 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs @@ -1,12 +1,11 @@ //! Location maps retaining the original compiled Python destination objects. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_project::resource_views::main::resource_reads::destination::destination; use sqlbuild_python_project::resource_views::main::resource_reads::name::name; -fn locations<'py>(items: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn locations<'py>(items: &Bound<'py, PyAny>) -> PyResult> { let result = PyDict::new(items.py()); for item in items.try_iter()? { let item = item?; @@ -14,28 +13,3 @@ fn locations<'py>(items: &Bound<'py, PyAny>) -> PyResult> { } Ok(result) } - -#[pyfunction] -pub(crate) fn build_model_locations<'py>( - models: &Bound<'py, PyAny>, -) -> PyResult> { - locations(models) -} - -#[pyfunction] -pub(crate) fn build_seed_locations<'py>(seeds: &Bound<'py, PyAny>) -> PyResult> { - locations(seeds) -} - -#[pyfunction] -pub(crate) fn build_function_locations<'py>( - functions: &Bound<'py, PyAny>, -) -> PyResult> { - locations(functions) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_model_locations, module)?)?; - module.add_function(wrap_pyfunction!(build_seed_locations, module)?)?; - module.add_function(wrap_pyfunction!(build_function_locations, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs index fe2394b45f..3460b890f7 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs @@ -1,13 +1,14 @@ //! Model, function and SQL-hook orchestration through adapter-owned rendering. -use crate::planner_resolution::_helpers::cursor::cursor_model::compute_model_cursor_bounds; -use crate::planner_resolution::_helpers::cursor::cursor_model::unbounded; -use crate::planner_resolution::models::CursorFiltering; -use crate::planner_resolution::models::ModelSqlRequest; +use crate::planner_resolution::_helpers::cursor::cursor_model::{ + compute_model_cursor_bounds, unbounded, +}; +use crate::planner_resolution::main::references::model_hook_entries::model_hook_entries; use crate::planner_resolution::models::Resolution; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::planner_resolution::models::{CursorFiltering, ModelSqlRequest}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{PyErr, pyfunction, wrap_pyfunction}; +use pyo3::{PyErr, pyfunction}; use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -20,18 +21,16 @@ use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; use sqlbuild_python_front::model_config::main::config::config_text_is::config_text_is; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; -use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; -use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; const APPEND_STRATEGY: &str = "append"; #[pyfunction(name = "resolve_model_sql")] #[pyo3(signature = (**kwargs))] -fn resolve_model_sql_of<'py>( +pub(crate) fn resolve_model_sql_of<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -180,50 +179,9 @@ fn unresolved_bounds_error(py: Python<'_>, name: &str, cause: Option<&str>) -> P Ok(PyErr::from_value(error.call((message,), Some(&kwargs))?)) } -#[pyfunction] -#[pyo3(signature = (**kwargs))] -pub(crate) fn resolve_function_sql<'py>( - py: Python<'py>, - kwargs: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "resolve_function_sql", - kwargs, - ( - &[ - "adapter", - "function", - "model_locations", - "seed_locations", - "function_locations", - "source_map", - "source_warehouse_columns", - "star_exclude_keyword", - ], - &[], - ), - )?; - let resolution = Resolution { - adapter: args.value("adapter")?, - external: py.None().into_bound(py), - source_map: args.value("source_map")?, - source_warehouse_columns: args.value("source_warehouse_columns")?, - model_locations: args.value("model_locations")?, - seed_locations: args.value("seed_locations")?, - function_locations: args.value("function_locations")?, - }; - let function = args.value("function")?; - let name: String = - sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)? - .extract()?; - let sql: String = resolution.unbounded(body_sql(&function)?, &format!("Function '{name}'"))?; - Ok(PyString::new(py, &sql).into_any()) -} - #[pyfunction] #[pyo3(signature = (*, model, adapter, context, external_sql_reference_resolver))] -fn resolve_model_hook_entries<'py>( +pub(crate) fn resolve_model_hook_entries<'py>( model: &Bound<'py, PyAny>, adapter: Bound<'py, PyAny>, context: Bound<'py, PyAny>, @@ -232,27 +190,3 @@ fn resolve_model_hook_entries<'py>( let resolution = Resolution::of_context(adapter, &context, external_sql_reference_resolver)?; model_hook_entries(model, &resolution) } - -/// One model's pre and post hooks with their SQL resolved. -pub(crate) fn model_hook_entries<'py>( - model: &Bound<'py, PyAny>, - resolution: &Resolution<'py>, -) -> PyResult> { - let py: Python<'py> = model.py(); - let values = model_values(model)?; - let name: String = model_name(model)?; - let mut hooks: Vec> = Vec::new(); - for key in ["pre_hooks", "post_hooks"] { - hooks.push(resolution.hook( - config_value(py, &values, key)?, - &format!("Model '{name}' {key}"), - )?); - } - PyTuple::new(model.py(), hooks) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_model_sql_of, module)?)?; - module.add_function(wrap_pyfunction!(resolve_function_sql, module)?)?; - module.add_function(wrap_pyfunction!(resolve_model_hook_entries, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs index a17cd50ced..23aff09270 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs @@ -2,15 +2,14 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::planner_resolution::_helpers::references::values::cursor_wrap; -use crate::planner_resolution::_helpers::references::values::filtering_of; -use crate::planner_resolution::_helpers::references::values::method; -use crate::planner_resolution::_helpers::references::values::string; +use crate::planner_resolution::_helpers::references::values::{ + cursor_wrap, filtering_of, method, string, +}; use crate::planner_resolution::models::CursorFiltering; use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -21,7 +20,10 @@ use sqlbuild_sqltext::resolution::main::{ref_alias::ref_alias, reference_spans:: #[pyfunction] #[pyo3(signature = (**kwargs))] -fn resolve_ref_references(py: Python<'_>, kwargs: Option>) -> PyResult { +pub(crate) fn resolve_ref_references( + py: Python<'_>, + kwargs: Option>, +) -> PyResult { let args = KeywordArguments::read( py, "resolve_ref_references", @@ -229,40 +231,3 @@ pub(crate) fn resolve_table_function_references( ) -> PyResult { resolve_function_calls(query_sql, function_locations, adapter, true) } - -#[pyfunction] -#[pyo3(signature = (*, model_locations, seed_locations, deferred_locations, selected_keys))] -pub(crate) fn apply_deferred_locations<'py>( - model_locations: &Bound<'py, PyDict>, - seed_locations: &Bound<'py, PyDict>, - deferred_locations: &Bound<'py, PyDict>, - selected_keys: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { - let selected = selected_keys - .try_iter()? - .map(|key| native_field(&key?, "name")?.extract::()) - .collect::>>()?; - let models = model_locations.copy()?; - let seeds = seed_locations.copy()?; - for (name, target) in deferred_locations.iter() { - if selected.contains(&name.extract::()?) { - continue; - } - if models.contains(&name)? { - models.set_item(&name, &target)?; - } - if seeds.contains(&name)? { - seeds.set_item(&name, &target)?; - } - } - Ok((models, seeds)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_ref_references, module)?)?; - module.add_function(wrap_pyfunction!(resolve_dbt_ref_references, module)?)?; - module.add_function(wrap_pyfunction!(resolve_udf_references, module)?)?; - module.add_function(wrap_pyfunction!(resolve_table_function_references, module)?)?; - module.add_function(wrap_pyfunction!(apply_deferred_locations, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs index 14f27651ad..a650168767 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs @@ -1,17 +1,14 @@ //! Source contract validation and adapter-controlled projection rendering. -use crate::planner_resolution::_helpers::references::values::cursor_wrap; -use crate::planner_resolution::_helpers::references::values::filtering_of; -use crate::planner_resolution::_helpers::references::values::method; -use crate::planner_resolution::_helpers::references::values::string; +use crate::planner_resolution::_helpers::references::values::{ + cursor_wrap, filtering_of, method, string, +}; use crate::planner_resolution::models::CursorFiltering; use crate::planner_strategy::main::values::input_error::input_error; use pyo3::IntoPyObject; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; @@ -287,7 +284,7 @@ fn resolve_one( #[pyfunction] #[pyo3(signature = (**kwargs))] -fn resolve_source_references( +pub(crate) fn resolve_source_references( py: Python<'_>, kwargs: Option>, ) -> PyResult { @@ -352,9 +349,3 @@ pub(crate) fn resolve_sources( output.push_str(&sql[last..]); Ok(output) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_source_references, module)?)?; - module.add_function(wrap_pyfunction!(render_source_read_relation, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/future_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/future_safety.rs index 7ef4b356af..7de00a6118 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/future_safety.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/future_safety.rs @@ -1,14 +1,11 @@ //! Future cursor safety evidence and exception boundary. -use crate::planner_resolution::_helpers::cursor::cursor_safety::horizon; -use crate::planner_resolution::_helpers::cursor::cursor_safety::replace; -use crate::planner_resolution::_helpers::cursor::cursor_safety::safety_duration; -use crate::planner_resolution::_helpers::cursor::cursor_safety::safety_error; -use crate::planner_resolution::_helpers::cursor::cursor_safety::timestamp_bound; -use crate::planner_resolution::_helpers::cursor::cursor_safety::utc_clock; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::planner_resolution::_helpers::cursor::cursor_safety::{ + horizon, replace, safety_duration, safety_error, timestamp_bound, utc_clock, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::main::next_boundary::next_boundary; use sqlbuild_cursor::algebra::main::render::render; @@ -166,7 +163,3 @@ pub(crate) fn resolve_future_cursor_safety<'py>( )?; replace(&bounds, &[("end", applied_end), ("future_safety", safety)]) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_future_cursor_safety, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs index 3c4f20b382..ff3fd29285 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs @@ -1,23 +1,17 @@ //! Maximum automatic-start recovery and evidence boundary. use crate::planner_resolution::_helpers::cursor::cursor_bounds::apply_typed_cursor_replay_policy; -use crate::planner_resolution::_helpers::cursor::cursor_bounds::optional_scalar; -use crate::planner_resolution::_helpers::cursor::cursor_bounds::required_scalar; -use crate::planner_resolution::_helpers::cursor::cursor_bounds::scalars; -use crate::planner_resolution::_helpers::cursor::cursor_safety::horizon; -use crate::planner_resolution::_helpers::cursor::cursor_safety::replace; -use crate::planner_resolution::_helpers::cursor::cursor_safety::safety_duration; -use crate::planner_resolution::_helpers::cursor::cursor_safety::safety_error; -use crate::planner_resolution::_helpers::cursor::cursor_safety::timestamp_bound; -use crate::planner_resolution::_helpers::cursor::cursor_safety::utc_clock; -use crate::planner_resolution::_helpers::references::values::string; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +use crate::planner_resolution::_helpers::cursor::cursor_bounds::{ + optional_scalar, required_scalar, scalars, +}; +use crate::planner_resolution::_helpers::cursor::cursor_safety::{ + horizon, replace, safety_duration, safety_error, timestamp_bound, utc_clock, }; +use crate::planner_resolution::_helpers::references::values::string; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyDict; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_cursor::algebra::main::compare::compare; -use sqlbuild_cursor::algebra::main::render::render; use sqlbuild_cursor::planner::main::eligible_start::eligible_start; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -33,47 +27,6 @@ use sqlbuild_python_front::cursor::models::CursorSession; use sqlbuild_python_project::planner_phases::main::object::object; use std::cmp::Ordering; -#[pyfunction] -#[pyo3(signature = (**kwargs))] -pub(crate) fn maximum_allowed_start<'py>( - py: Python<'py>, - kwargs: Option>, -) -> PyResult { - let args = KeywordArguments::read( - py, - "maximum_allowed_start", - kwargs, - ( - &[ - "discovered_value", - "cursor_grain", - "invocation_time", - "max_ahead", - ], - &[], - ), - )?; - let session = CursorSession::new(py)?; - let discovered = required_scalar( - &session, - &args.value("discovered_value")?, - &string(py, "timestamp"), - )?; - let duration = safety_duration(&args.value("max_ahead")?, false)?; - session.checked(render( - &horizon( - &session, - &session.python_scalar(&discovered)?, - ( - &args.value("invocation_time")?, - &args.value("cursor_grain")?, - ), - &duration, - )?, - &session, - )) -} - #[pyfunction] #[pyo3(signature = (**kwargs))] pub(crate) fn apply_maximum_start_policy<'py>( @@ -287,8 +240,3 @@ fn exceeded_message(evidence: &Bound<'_, PyAny>, non_idempotent: bool) -> PyResu native_field(evidence, "cursor_column")?.str()?, )) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(maximum_allowed_start, module)?)?; - module.add_function(wrap_pyfunction!(apply_maximum_start_policy, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/config_str.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/config_str.rs index 862fa084a4..673dbe2d6d 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/config_str.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/config_str.rs @@ -1,12 +1,12 @@ //! One string-valued model configuration entry. -use crate::planner_resolution::_helpers::cursor::cursor_ranges; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; /// One string-valued model configuration entry. -#[inline] pub fn config_str<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { - cursor_ranges::config_str(model, key) + let py: Python<'py> = model.py(); + Ok(config_text_object(py, &model_values(model)?, key)? + .unwrap_or_else(|| py.None().into_bound(py))) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/cursor_safety_configs.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/cursor_safety_configs.rs index 8bac8ea563..814d8fdea0 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/cursor_safety_configs.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/cursor_safety_configs.rs @@ -1,15 +1,25 @@ //! The future-end and automatic-start policies effective for one model. -use crate::planner_resolution::_helpers::cursor::cursor_ranges; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_phases::main::inputs::class_field::class_field; +use crate::planner_resolution::_helpers::cursor::cursor_policies::{ + future_cursor_config, start_cursor_config, +}; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// The future-end and automatic-start policies effective for one model. -#[inline] pub fn cursor_safety_configs<'py>( model: &Bound<'py, PyAny>, context: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - cursor_ranges::cursor_safety_configs(model, context) + Ok(( + future_cursor_config( + model, + &class_field(context, |c: &ModelPlanContext| &c.future_cursor_config)?, + )?, + start_cursor_config( + model, + &class_field(context, |c: &ModelPlanContext| &c.start_cursor_config)?, + )?, + )) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/empty_cursor_inputs.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/empty_cursor_inputs.rs index 93ab8f5f15..8ca3bef534 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/empty_cursor_inputs.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/empty_cursor_inputs.rs @@ -1,16 +1,39 @@ //! Empty inputs that leave a planner-owned window empty, and whether others have rows. -use crate::planner_resolution::_helpers::cursor::cursor_ranges; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::INCREMENTAL; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::ROLLING_WINDOW; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::model_cursor_snapshot; +use crate::planner_resolution::_helpers::cursor::cursor_windows::cursor_inputs_without_rows; +use crate::planner_resolution::main::cursor::config_str::config_str; use crate::planner_resolution::models::CursorWindowInputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Empty inputs that leave a planner-owned window empty, and whether others have rows. -#[inline] pub fn empty_cursor_inputs<'py>( model: &Bound<'py, PyAny>, inputs: &CursorWindowInputs<'py>, ) -> PyResult<(Bound<'py, PyAny>, bool)> { - cursor_ranges::empty_cursor_inputs(model, inputs) + let py = model.py(); + let nothing = + || -> PyResult<(Bound<'py, PyAny>, bool)> { Ok((PyTuple::empty(py).into_any(), false)) }; + if inputs.full_refresh + || inputs.runtime_owned + || !config_str(model, "materialized")?.eq(INCREMENTAL)? + || config_str(model, "cursor")?.is_none() + || config_str(model, "microbatch_strategy")?.eq(ROLLING_WINDOW)? + || (!inputs.overrides.0.is_none() && !inputs.overrides.1.is_none()) + { + return nothing(); + } + let cursor_snapshot = model_cursor_snapshot(model, &inputs.snapshot)?; + if cursor_snapshot.is_none() { + return nothing(); + } + let decision = cursor_inputs_without_rows(&cursor_snapshot)?; + Ok(( + native_field(&decision, "input_names")?, + native_field(&decision, "waiting_on_empty_inputs")?.is_truthy()?, + )) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/microbatch_range.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/microbatch_range.rs index b370ffbb28..86132f1819 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/microbatch_range.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/microbatch_range.rs @@ -2,9 +2,7 @@ use crate::planner_resolution::_helpers::cursor::cursor_ranges; use crate::planner_resolution::models::CursorWindowInputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Compute the real overall cursor range for microbatch batch splitting. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/model_cursor_overrides.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/model_cursor_overrides.rs index fa58db9799..12a86e1137 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/model_cursor_overrides.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/model_cursor_overrides.rs @@ -1,16 +1,78 @@ //! Resolve typed cursor overrides to generic start/end for one model. -use crate::planner_resolution::_helpers::cursor::cursor_ranges; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::INTEGER; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::TIMESTAMP; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::compile_input_error; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::config_bound; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::order; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::token; +use crate::planner_resolution::main::cursor::config_str::config_str; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::cursor::models::CursorSession; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use std::cmp::Ordering; /// Resolve typed cursor overrides to generic start/end for one model. -#[inline] pub fn model_cursor_overrides<'py>( model: &Bound<'py, PyAny>, cursor_overrides: &Bound<'py, PyAny>, (start, end): (Bound<'py, PyAny>, Bound<'py, PyAny>), ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - cursor_ranges::model_cursor_overrides(model, cursor_overrides, (start, end)) + let py = model.py(); + let cursor_type = config_str(model, "cursor_type")?; + let (mut resolved_start, mut resolved_end) = (start, end); + let typed_fields = if cursor_overrides.is_none() { + None + } else if cursor_type.eq(TIMESTAMP)? { + Some(("start_ts", "end_ts")) + } else if cursor_type.eq(INTEGER)? { + Some(("start_int", "end_int")) + } else { + None + }; + if let Some((start_field, end_field)) = typed_fields { + let typed_start = native_field(cursor_overrides, start_field)?; + if !typed_start.is_none() { + resolved_start = token(py, &typed_start)?; + } + let typed_end = native_field(cursor_overrides, end_field)?; + if !typed_end.is_none() { + resolved_end = token(py, &typed_end)?; + } + } + let session = CursorSession::new(py)?; + let name = name_object(model)?.str()?; + let cursor_start = config_bound(model, "cursor_start")?; + let cursor_end = config_bound(model, "cursor_end")?; + if !resolved_start.is_none() + && !cursor_start.is_none() + && order(&session, (&resolved_start, &cursor_start), &cursor_type)? == Ordering::Less + { + return Err(compile_input_error( + py, + format!( + "model '{name}': explicit start {} is before cursor_start {}", + resolved_start.str()?, + cursor_start.str()? + ), + )); + } + if !cursor_end.is_none() { + for (value, label) in [(&resolved_start, "start"), (&resolved_end, "end")] { + if !value.is_none() + && order(&session, (value, &cursor_end), &cursor_type)? != Ordering::Less + { + return Err(compile_input_error( + py, + format!( + "model '{name}': explicit {label} {} is at or beyond exclusive cursor_end {}", + value.str()?, + cursor_end.str()? + ), + )); + } + } + } + Ok((resolved_start, resolved_end)) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/normalize_cursor_snapshot_grain.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/normalize_cursor_snapshot_grain.rs index 0562231cb4..e9465f70cc 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/normalize_cursor_snapshot_grain.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/normalize_cursor_snapshot_grain.rs @@ -1,9 +1,7 @@ //! Normalize cursor snapshot grain. use crate::planner_resolution::_helpers::cursor::cursor_windows; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Normalize cursor snapshot grain. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/plan_cursor_bounds.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/plan_cursor_bounds.rs index 193df91965..050b6e8cb9 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/plan_cursor_bounds.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/plan_cursor_bounds.rs @@ -1,16 +1,55 @@ //! Compute cursor bounds for inclusion on the plan entry. -use crate::planner_resolution::_helpers::cursor::cursor_ranges; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::BoundsRequest; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::FULL_BACKFILL; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::INCREMENTAL; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::MICROBATCH; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::bounded_override; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::clamp_cursor_end; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::model_cursor_snapshot; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::snapshot_bounds; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::without_destination; +use crate::planner_resolution::main::cursor::config_str::config_str; use crate::planner_resolution::models::CursorWindowInputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Compute cursor bounds for inclusion on the plan entry. -#[inline] pub fn plan_cursor_bounds<'py>( model: &Bound<'py, PyAny>, inputs: &CursorWindowInputs<'py>, ) -> PyResult> { - cursor_ranges::plan_cursor_bounds(model, inputs) + let py = model.py(); + if !config_str(model, "materialized")?.eq(INCREMENTAL)? + || config_str(model, "cursor")?.is_none() + { + return Ok(py.None().into_bound(py)); + } + let bounded = bounded_override(model, inputs)?; + if !bounded.is_none() { + return clamp_cursor_end(bounded, model); + } + if inputs.runtime_owned || inputs.full_refresh { + return Ok(py.None().into_bound(py)); + } + let mut cursor_snapshot = model_cursor_snapshot(model, &inputs.snapshot)?; + if cursor_snapshot.is_none() { + return Ok(cursor_snapshot); + } + if native_field(&inputs.backfill, "action")?.eq(FULL_BACKFILL)? { + cursor_snapshot = without_destination(cursor_snapshot)?; + } + snapshot_bounds( + model, + inputs, + BoundsRequest { + cursor_snapshot, + typing: ( + config_str(model, "cursor_type")?, + config_str(model, "cursor_grain")?, + ), + lookback: config_str(model, "lookback")?, + is_microbatch: config_str(model, "incremental_mode")?.eq(MICROBATCH)?, + }, + ) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_bounds.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_bounds.rs index 830c594d63..8b84cfd621 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_bounds.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_bounds.rs @@ -1,12 +1,12 @@ //! Register the cursor bounds functions and classes. -use crate::planner_resolution::_helpers::cursor::cursor_bounds; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::apply_typed_cursor_replay_policy; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::compute_cursor_bounds; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the cursor bounds functions and classes. -#[inline] pub fn register_cursor_bounds(module: &Bound<'_, PyModule>) -> PyResult<()> { - cursor_bounds::register(module) + module.add_function(wrap_pyfunction!(compute_cursor_bounds, module)?)?; + module.add_function(wrap_pyfunction!(apply_typed_cursor_replay_policy, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_overrides.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_overrides.rs index 5b1d19dab6..78ed64fb26 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_overrides.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_overrides.rs @@ -1,12 +1,10 @@ //! Register the cursor overrides functions and classes. -use crate::planner_resolution::_helpers::cursor::cursor_overrides; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::cursor::cursor_overrides::type_cursor_overrides_of; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the cursor overrides functions and classes. -#[inline] pub fn register_cursor_overrides(module: &Bound<'_, PyModule>) -> PyResult<()> { - cursor_overrides::register(module) + module.add_function(wrap_pyfunction!(type_cursor_overrides_of, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs index 85c893558d..ee4dee1ca0 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs @@ -1,12 +1,14 @@ //! Register the cursor ranges functions and classes. -use crate::planner_resolution::_helpers::cursor::cursor_ranges; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::compute_microbatch_range; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::compute_plan_cursor_bounds; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::resolve_cursor_overrides; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the cursor ranges functions and classes. -#[inline] pub fn register_cursor_ranges(module: &Bound<'_, PyModule>) -> PyResult<()> { - cursor_ranges::register(module) + module.add_function(wrap_pyfunction!(compute_plan_cursor_bounds, module)?)?; + module.add_function(wrap_pyfunction!(compute_microbatch_range, module)?)?; + module.add_function(wrap_pyfunction!(resolve_cursor_overrides, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_warnings.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_warnings.rs index f0c033ae7d..7680ec73cc 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_warnings.rs @@ -1,12 +1,12 @@ //! Register the cursor warnings functions and classes. -use crate::planner_resolution::_helpers::cursor::cursor_warnings; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::cursor::cursor_warnings::future_cursor_cap_warning; +use crate::planner_resolution::_helpers::cursor::cursor_warnings::maximum_start_cap_warning; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the cursor warnings functions and classes. -#[inline] pub fn register_cursor_warnings(module: &Bound<'_, PyModule>) -> PyResult<()> { - cursor_warnings::register(module) + module.add_function(wrap_pyfunction!(future_cursor_cap_warning, module)?)?; + module.add_function(wrap_pyfunction!(maximum_start_cap_warning, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_windows.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_windows.rs index ddda2d8f12..bb188ec60e 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_windows.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_windows.rs @@ -1,12 +1,20 @@ //! Register the cursor windows functions and classes. -use crate::planner_resolution::_helpers::cursor::cursor_windows; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::cursor::cursor_windows::cursor_window_override_flags; +use crate::planner_resolution::_helpers::cursor::cursor_windows::decide_empty_cursor_inputs; +use crate::planner_resolution::_helpers::cursor::cursor_windows::empty_cursor_window; +use crate::planner_resolution::_helpers::cursor::cursor_windows::empty_input_rebuild_refusal; +use crate::planner_resolution::_helpers::cursor::cursor_windows::normalize_cursor_snapshot_grain; +use crate::planner_resolution::main::cursor::resolve_effective_timestamp_grain::resolve_effective_timestamp_grain; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the cursor windows functions and classes. -#[inline] pub fn register_cursor_windows(module: &Bound<'_, PyModule>) -> PyResult<()> { - cursor_windows::register(module) + module.add_function(wrap_pyfunction!(decide_empty_cursor_inputs, module)?)?; + module.add_function(wrap_pyfunction!(empty_cursor_window, module)?)?; + module.add_function(wrap_pyfunction!(cursor_window_override_flags, module)?)?; + module.add_function(wrap_pyfunction!(empty_input_rebuild_refusal, module)?)?; + module.add_function(wrap_pyfunction!(resolve_effective_timestamp_grain, module)?)?; + module.add_function(wrap_pyfunction!(normalize_cursor_snapshot_grain, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_effective_timestamp_grain.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_effective_timestamp_grain.rs index 6e492dc47e..18e239e2d7 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_effective_timestamp_grain.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_effective_timestamp_grain.rs @@ -1,22 +1,41 @@ //! Resolve effective timestamp grain. -use crate::planner_resolution::_helpers::cursor::cursor_windows; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::references::values::string; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_front::cursor::models::CursorSession; -/// Resolve effective timestamp grain. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, cursor_type, downstream_grain, cursor_input_grains, microbatch_strategy=None))] pub fn resolve_effective_timestamp_grain<'py>( cursor_type: &Bound<'py, PyAny>, downstream_grain: &Bound<'py, PyAny>, cursor_input_grains: &Bound<'py, PyAny>, microbatch_strategy: Option<&str>, ) -> PyResult> { - cursor_windows::resolve_effective_timestamp_grain( - cursor_type, - downstream_grain, - cursor_input_grains, - microbatch_strategy, - ) + let py = cursor_type.py(); + if !cursor_type.eq("timestamp")? { + return Ok(py.None().into_bound(py)); + } + let mut effective = if downstream_grain.is_truthy()? { + downstream_grain.clone() + } else { + string(py, "second") + }; + if microbatch_strategy == Some("watermark") { + return Ok(effective); + } + let session = CursorSession::new(py)?; + for value in cursor_input_grains.try_iter()? { + let value = value?; + let candidate = if value.is_truthy()? { + value + } else { + string(py, "second") + }; + if session.grain(&candidate)? > session.grain(&effective)? { + effective = candidate; + } + } + Ok(effective) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_start_cursor_config.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_start_cursor_config.rs index 60e0697ee1..2e6a9c6002 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_start_cursor_config.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_start_cursor_config.rs @@ -1,15 +1,12 @@ //! Resolve automatic-start policy with model config taking precedence. -use crate::planner_resolution::_helpers::cursor::cursor_policies; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::cursor::cursor_policies::start_cursor_config; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Resolve automatic-start policy with model config taking precedence. -#[inline] pub fn resolve_start_cursor_config<'py>( model: &Bound<'py, PyAny>, project_config: &Bound<'py, PyAny>, ) -> PyResult> { - cursor_policies::resolve_start_cursor_config(model, project_config) + start_cursor_config(model, project_config) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/type_cursor_overrides.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/type_cursor_overrides.rs index f49f4b313b..9d377e504c 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/type_cursor_overrides.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/type_cursor_overrides.rs @@ -1,9 +1,7 @@ //! Untyped cursor overrides typed by the cursor type of the selected incremental models. use crate::planner_resolution::_helpers::cursor::cursor_overrides; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Untyped cursor overrides typed by the cursor type of the selected incremental models. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/with_cursor_safety_warnings.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/with_cursor_safety_warnings.rs index ac4e3f080d..0f6a337dc8 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/with_cursor_safety_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/with_cursor_safety_warnings.rs @@ -1,16 +1,44 @@ //! `warnings` followed by the future-end and automatic-start warnings `bounds` carries. -use crate::planner_resolution::_helpers::cursor::cursor_warnings; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::cursor::cursor_warnings::FUTURE_CAPPED_CODE; +use crate::planner_resolution::_helpers::cursor::cursor_warnings::START_CAPPED_CODE; +use crate::planner_resolution::_helpers::cursor::cursor_warnings::future_cap_message; +use crate::planner_resolution::_helpers::cursor::cursor_warnings::maximum_start_message; +use crate::planner_resolution::_helpers::references::values::string; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; /// `warnings` followed by the future-end and automatic-start warnings `bounds` carries. -#[inline] pub fn with_cursor_safety_warnings<'py>( warnings: &Bound<'py, PyAny>, model_name: &Bound<'py, PyAny>, bounds: &Bound<'py, PyAny>, ) -> PyResult> { - cursor_warnings::with_cursor_safety_warnings(warnings, model_name, bounds) + let py = warnings.py(); + let mut all: Vec> = warnings + .try_iter()? + .collect::>>>()?; + for (message, code) in [ + (future_cap_message(bounds)?, FUTURE_CAPPED_CODE), + (maximum_start_message(bounds)?, START_CAPPED_CODE), + ] { + let Some(message) = message else { continue }; + all.push(new_record_with( + py, + RecordId::PlanWarning, + &[ + ("model_name", model_name.clone()), + ( + "severity", + enum_member(py, RecordId::WarningSeverity, "WARNING")?, + ), + ("code", string(py, code)), + ("message", string(py, &message)), + ], + )?); + } + Ok(PyTuple::new(py, all)?.into_any()) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/apply_deferred_locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/apply_deferred_locations.rs index 28529111fa..02f0a9ddea 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/apply_deferred_locations.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/apply_deferred_locations.rs @@ -1,23 +1,34 @@ //! Apply deferred locations. -use crate::planner_resolution::_helpers::references::refs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; -/// Apply deferred locations. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, model_locations, seed_locations, deferred_locations, selected_keys))] pub fn apply_deferred_locations<'py>( model_locations: &Bound<'py, PyDict>, seed_locations: &Bound<'py, PyDict>, deferred_locations: &Bound<'py, PyDict>, selected_keys: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { - refs::apply_deferred_locations( - model_locations, - seed_locations, - deferred_locations, - selected_keys, - ) + let selected = selected_keys + .try_iter()? + .map(|key| native_field(&key?, "name")?.extract::()) + .collect::>>()?; + let models = model_locations.copy()?; + let seeds = seed_locations.copy()?; + for (name, target) in deferred_locations.iter() { + if selected.contains(&name.extract::()?) { + continue; + } + if models.contains(&name)? { + models.set_item(&name, &target)?; + } + if seeds.contains(&name)? { + seeds.set_item(&name, &target)?; + } + } + Ok((models, seeds)) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_function_locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_function_locations.rs index 583e56dda5..19045ab702 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_function_locations.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_function_locations.rs @@ -1,15 +1,13 @@ //! Build function locations. -use crate::planner_resolution::_helpers::references::locations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::references::locations::locations; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyDict; -/// Build function locations. -#[inline] +#[pyfunction] pub fn build_function_locations<'py>( functions: &Bound<'py, PyAny>, ) -> PyResult> { - locations::build_function_locations(functions) + locations(functions) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_model_locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_model_locations.rs index 674abc8aea..f71e189c57 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_model_locations.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_model_locations.rs @@ -1,13 +1,11 @@ //! Build model locations. -use crate::planner_resolution::_helpers::references::locations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::references::locations::locations; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyDict; -/// Build model locations. -#[inline] +#[pyfunction] pub fn build_model_locations<'py>(models: &Bound<'py, PyAny>) -> PyResult> { - locations::build_model_locations(models) + locations(models) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_seed_locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_seed_locations.rs index 9df886fb51..68a222d3cb 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_seed_locations.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_seed_locations.rs @@ -1,13 +1,11 @@ //! Build seed locations. -use crate::planner_resolution::_helpers::references::locations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::references::locations::locations; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyDict; -/// Build seed locations. -#[inline] +#[pyfunction] pub fn build_seed_locations<'py>(seeds: &Bound<'py, PyAny>) -> PyResult> { - locations::build_seed_locations(seeds) + locations(seeds) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/model_hook_entries.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/model_hook_entries.rs index 906c263d9a..1a1c3e3c93 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/model_hook_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/model_hook_entries.rs @@ -1,17 +1,26 @@ //! One model's pre and post hooks with their SQL resolved. -use crate::planner_resolution::_helpers::references::orchestration; use crate::planner_resolution::models::Resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyTuple; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; /// One model's pre and post hooks with their SQL resolved. -#[inline] pub fn model_hook_entries<'py>( model: &Bound<'py, PyAny>, resolution: &Resolution<'py>, ) -> PyResult> { - orchestration::model_hook_entries(model, resolution) + let py: Python<'py> = model.py(); + let values = model_values(model)?; + let name: String = model_name(model)?; + let mut hooks: Vec> = Vec::new(); + for key in ["pre_hooks", "post_hooks"] { + hooks.push(resolution.hook( + config_value(py, &values, key)?, + &format!("Model '{name}' {key}"), + )?); + } + PyTuple::new(model.py(), hooks) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs index ec44ce6a56..edd12f9299 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs @@ -1,12 +1,14 @@ //! Register the locations functions and classes. -use crate::planner_resolution::_helpers::references::locations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_resolution::main::references::build_function_locations::build_function_locations; +use crate::planner_resolution::main::references::build_model_locations::build_model_locations; +use crate::planner_resolution::main::references::build_seed_locations::build_seed_locations; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the locations functions and classes. -#[inline] pub fn register_locations(module: &Bound<'_, PyModule>) -> PyResult<()> { - locations::register(module) + module.add_function(wrap_pyfunction!(build_model_locations, module)?)?; + module.add_function(wrap_pyfunction!(build_seed_locations, module)?)?; + module.add_function(wrap_pyfunction!(build_function_locations, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs index 194040359c..3934e61676 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs @@ -1,12 +1,14 @@ //! Register the orchestration functions and classes. -use crate::planner_resolution::_helpers::references::orchestration; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::references::orchestration::resolve_model_hook_entries; +use crate::planner_resolution::_helpers::references::orchestration::resolve_model_sql_of; +use crate::planner_resolution::main::references::resolve_function_sql::resolve_function_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the orchestration functions and classes. -#[inline] pub fn register_orchestration(module: &Bound<'_, PyModule>) -> PyResult<()> { - orchestration::register(module) + module.add_function(wrap_pyfunction!(resolve_model_sql_of, module)?)?; + module.add_function(wrap_pyfunction!(resolve_function_sql, module)?)?; + module.add_function(wrap_pyfunction!(resolve_model_hook_entries, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs index 368e042aa8..214f231b20 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs @@ -1,12 +1,19 @@ //! Register the refs functions and classes. -use crate::planner_resolution::_helpers::references::refs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::references::refs::resolve_dbt_ref_references; +use crate::planner_resolution::_helpers::references::refs::resolve_ref_references; +use crate::planner_resolution::_helpers::references::refs::resolve_table_function_references; +use crate::planner_resolution::_helpers::references::refs::resolve_udf_references; +use crate::planner_resolution::main::references::apply_deferred_locations::apply_deferred_locations; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the refs functions and classes. -#[inline] pub fn register_refs(module: &Bound<'_, PyModule>) -> PyResult<()> { - refs::register(module) + module.add_function(wrap_pyfunction!(resolve_ref_references, module)?)?; + module.add_function(wrap_pyfunction!(resolve_dbt_ref_references, module)?)?; + module.add_function(wrap_pyfunction!(resolve_udf_references, module)?)?; + module.add_function(wrap_pyfunction!(resolve_table_function_references, module)?)?; + module.add_function(wrap_pyfunction!(apply_deferred_locations, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_sources.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_sources.rs index 37f8b61408..5fba9ca4f4 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_sources.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_sources.rs @@ -1,12 +1,13 @@ //! Register the sources functions and classes. -use crate::planner_resolution::_helpers::references::sources; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::references::sources::render_source_read_relation; +use crate::planner_resolution::_helpers::references::sources::resolve_source_references; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the sources functions and classes. -#[inline] pub fn register_sources(module: &Bound<'_, PyModule>) -> PyResult<()> { - sources::register(module) + module.add_function(wrap_pyfunction!(resolve_source_references, module)?)?; + module.add_function(wrap_pyfunction!(render_source_read_relation, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_function_sql.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_function_sql.rs index df5e846fae..24046a923e 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_function_sql.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_function_sql.rs @@ -1,17 +1,49 @@ //! Resolve function sql. -use crate::planner_resolution::_helpers::references::orchestration; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; +use crate::planner_resolution::models::Resolution; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyString}; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; -/// Resolve function sql. -#[inline] +#[pyfunction] +#[pyo3(signature = (**kwargs))] pub fn resolve_function_sql<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { - orchestration::resolve_function_sql(py, kwargs) + let args = KeywordArguments::read( + py, + "resolve_function_sql", + kwargs, + ( + &[ + "adapter", + "function", + "model_locations", + "seed_locations", + "function_locations", + "source_map", + "source_warehouse_columns", + "star_exclude_keyword", + ], + &[], + ), + )?; + let resolution = Resolution { + adapter: args.value("adapter")?, + external: py.None().into_bound(py), + source_map: args.value("source_map")?, + source_warehouse_columns: args.value("source_warehouse_columns")?, + model_locations: args.value("model_locations")?, + seed_locations: args.value("seed_locations")?, + function_locations: args.value("function_locations")?, + }; + let function = args.value("function")?; + let name: String = + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)? + .extract()?; + let sql: String = resolution.unbounded(body_sql(&function)?, &format!("Function '{name}'"))?; + Ok(PyString::new(py, &sql).into_any()) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_model_sql.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_model_sql.rs index cefc089062..2747df8f56 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_model_sql.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_model_sql.rs @@ -3,9 +3,7 @@ use crate::planner_resolution::_helpers::references::orchestration; use crate::planner_resolution::models::ModelSqlRequest; use crate::planner_resolution::models::Resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// One model's planned SQL: references resolved, cursor inputs filtered and intrinsics bound. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/maximum_allowed_start.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/maximum_allowed_start.rs index acae9f4346..caaae5d68c 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/maximum_allowed_start.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/maximum_allowed_start.rs @@ -1,16 +1,52 @@ //! Maximum allowed start. -use crate::planner_resolution::_helpers::safety::maximum_safety; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::required_scalar; +use crate::planner_resolution::_helpers::cursor::cursor_safety::{horizon, safety_duration}; +use crate::planner_resolution::_helpers::references::values::string; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_cursor::algebra::main::render::render; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_front::cursor::models::CursorSession; -/// Maximum allowed start. -#[inline] +#[pyfunction] +#[pyo3(signature = (**kwargs))] pub fn maximum_allowed_start<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult { - maximum_safety::maximum_allowed_start(py, kwargs) + let args = KeywordArguments::read( + py, + "maximum_allowed_start", + kwargs, + ( + &[ + "discovered_value", + "cursor_grain", + "invocation_time", + "max_ahead", + ], + &[], + ), + )?; + let session = CursorSession::new(py)?; + let discovered = required_scalar( + &session, + &args.value("discovered_value")?, + &string(py, "timestamp"), + )?; + let duration = safety_duration(&args.value("max_ahead")?, false)?; + session.checked(render( + &horizon( + &session, + &session.python_scalar(&discovered)?, + ( + &args.value("invocation_time")?, + &args.value("cursor_grain")?, + ), + &duration, + )?, + &session, + )) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_future_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_future_safety.rs index 64dafdc018..4884879308 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_future_safety.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_future_safety.rs @@ -1,12 +1,10 @@ //! Register the future safety functions and classes. -use crate::planner_resolution::_helpers::safety::future_safety; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::safety::future_safety::resolve_future_cursor_safety; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the future safety functions and classes. -#[inline] pub fn register_future_safety(module: &Bound<'_, PyModule>) -> PyResult<()> { - future_safety::register(module) + module.add_function(wrap_pyfunction!(resolve_future_cursor_safety, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_maximum_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_maximum_safety.rs index f912b1d4a6..7d50ed639c 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_maximum_safety.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_maximum_safety.rs @@ -1,12 +1,12 @@ //! Register the maximum safety functions and classes. -use crate::planner_resolution::_helpers::safety::maximum_safety; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_resolution::_helpers::safety::maximum_safety::apply_maximum_start_policy; +use crate::planner_resolution::main::safety::maximum_allowed_start::maximum_allowed_start; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the maximum safety functions and classes. -#[inline] pub fn register_maximum_safety(module: &Bound<'_, PyModule>) -> PyResult<()> { - maximum_safety::register(module) + module.add_function(wrap_pyfunction!(maximum_allowed_start, module)?)?; + module.add_function(wrap_pyfunction!(apply_maximum_start_policy, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/models.rs b/crates/sqlbuild-python-planner/src/planner_resolution/models.rs index 17f70498bc..7db929d5a7 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/models.rs @@ -7,10 +7,9 @@ use crate::planner_resolution::_helpers::references::refs::resolve_refs; use crate::planner_resolution::_helpers::references::refs::resolve_table_function_references; use crate::planner_resolution::_helpers::references::refs::resolve_udf_references; use crate::planner_resolution::_helpers::references::sources::resolve_sources; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; use pyo3::prelude::PyAnyMethods; use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny}; use pyo3::types::PyDict; use pyo3::types::PyDictMethods; use pyo3::types::PyList; diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs index 3cb657c7f8..ed30375b64 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs @@ -1,10 +1,10 @@ //! Scenario artifact dataclasses and adapter identifier callbacks. -use crate::planner_scenario::_helpers::values::error; -use crate::planner_scenario::_helpers::values::string; -use crate::planner_scenario::constants::SCENARIO_PLAN_HASH_COLLISION; -use crate::planner_scenario::constants::SCENARIO_PLAN_INVALID_HASH_PREFIX; -use crate::planner_scenario::constants::SCENARIO_PLAN_RELATION_COLLISION; +use crate::planner_scenario::_helpers::values::{error, string}; +use crate::planner_scenario::constants::{ + SCENARIO_PLAN_HASH_COLLISION, SCENARIO_PLAN_INVALID_HASH_PREFIX, + SCENARIO_PLAN_RELATION_COLLISION, +}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_scenario::main::{hash_prefix, parse_artifact_name}; @@ -176,11 +176,6 @@ pub(crate) fn parse_scenario_artifact_physical_name<'py>( ) } -#[pyfunction] -pub(crate) fn is_scenario_artifact_physical_name(name: &str) -> bool { - parse_artifact_name::parse_artifact_name(name).is_some() -} - #[pyfunction] #[pyo3(signature = (**kwargs))] pub(crate) fn build_scenario_relation_map<'py>( @@ -266,45 +261,3 @@ pub(crate) fn build_scenario_relation_map<'py>( ], ) } - -/// `build_scenario_artifact_name` as a function of `module`. -pub(crate) fn build_scenario_artifact_name_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_scenario_artifact_name, module) -} - -/// `build_scenario_hash_index` as a function of `module`. -pub(crate) fn build_scenario_hash_index_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_scenario_hash_index, module) -} - -/// `build_scenario_relation_map` as a function of `module`. -pub(crate) fn build_scenario_relation_map_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_scenario_relation_map, module) -} - -/// `compute_scenario_hash_prefix` as a function of `module`. -pub(crate) fn compute_scenario_hash_prefix_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(compute_scenario_hash_prefix, module) -} - -/// `is_scenario_artifact_physical_name` as a function of `module`. -pub(crate) fn scenario_artifact_physical_name_check_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(is_scenario_artifact_physical_name, module) -} - -/// `parse_scenario_artifact_physical_name` as a function of `module`. -pub(crate) fn parse_scenario_artifact_physical_name_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(parse_scenario_artifact_physical_name, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs index 93f9447935..372837a2b1 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs @@ -190,10 +190,3 @@ fn raise_for_warnings(warnings: &Bound<'_, PyAny>) -> PyResult<()> { } Ok(()) } - -/// `build_cli_scenario_plan` as a function of `module`. -pub(crate) fn build_cli_scenario_plan_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_cli_scenario_plan, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs index 6b97bb5ce5..46c3b16a06 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs @@ -14,14 +14,11 @@ use crate::planner_scenario::_helpers::sql::resolve_sql; use crate::planner_scenario::_helpers::sql::source_label; use crate::planner_scenario::_helpers::values::error; use crate::planner_scenario::_helpers::values::string; -use crate::planner_scenario::constants::CURSOR_INTRINSIC_NAMES; -use crate::planner_scenario::constants::CURSOR_TYPE_KEY; -use crate::planner_scenario::constants::INCREMENTAL_MODE_KEY; -use crate::planner_scenario::constants::INTEGER_CURSOR_TYPE; -use crate::planner_scenario::constants::MICROBATCH_INCREMENTAL_MODE; -use crate::planner_scenario::constants::SCENARIO_EXPECTED_COLUMNS_HELP; -use crate::planner_scenario::constants::SCENARIO_PLAN_EXPECTED_COLUMNS; -use crate::planner_scenario::constants::SCENARIO_PLAN_INVALID_FIXTURE; +use crate::planner_scenario::constants::{ + CURSOR_INTRINSIC_NAMES, CURSOR_TYPE_KEY, INCREMENTAL_MODE_KEY, INTEGER_CURSOR_TYPE, + MICROBATCH_INCREMENTAL_MODE, SCENARIO_EXPECTED_COLUMNS_HELP, SCENARIO_PLAN_EXPECTED_COLUMNS, + SCENARIO_PLAN_INVALID_FIXTURE, +}; use crate::planner_scenario::models::SqlContext; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; @@ -447,10 +444,3 @@ fn scenario_cursor_window<'py>(model: &Bound<'py, PyAny>) -> PyResult( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_scenario_execution_plan, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/fixtures.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/fixtures.rs index 4cd3040cdd..32509765e2 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/fixtures.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/fixtures.rs @@ -191,10 +191,3 @@ pub(crate) fn build_scenario_fixture_plans<'py>( &args.value("fixture_sql_overrides")?, ) } - -/// `build_scenario_fixture_plans` as a function of `module`. -pub(crate) fn build_scenario_fixture_plans_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_scenario_fixture_plans, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/graph.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/graph.rs index 36ac0ad16d..a1cb11e6d3 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/graph.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/graph.rs @@ -116,10 +116,3 @@ pub(crate) fn plan_scenario_graph<'py>( } Ok((plan, PyTuple::new(py, warnings)?)) } - -/// `plan_scenario_graph` as a function of `module`. -pub(crate) fn plan_scenario_graph_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(plan_scenario_graph, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/relations.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/relations.rs index 7d29b4f4d5..035c44aca4 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/relations.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/relations.rs @@ -1,10 +1,10 @@ //! Build scenario relation locations through adapter qualification callbacks. use crate::planner_scenario::_helpers::values::error; -use crate::planner_scenario::constants::REF; -use crate::planner_scenario::constants::SCENARIO_PLAN_INTERNAL; -use crate::planner_scenario::constants::SCENARIO_PLAN_MISSING_RELATION_TARGET; -use crate::planner_scenario::constants::SOURCE; +use crate::planner_scenario::constants::{REF, SOURCE}; +use crate::planner_scenario::constants::{ + SCENARIO_PLAN_INTERNAL, SCENARIO_PLAN_MISSING_RELATION_TARGET, +}; use crate::planner_scenario::models::ScenarioRelations; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods}; @@ -184,10 +184,3 @@ fn target_for_artifact<'py>( ], ) } - -/// `build_scenario_relation_plan_of` as a function of `module`. -pub(crate) fn build_scenario_relation_plan_of_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_scenario_relation_plan_of, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs index 85fc59560f..700030b8e8 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs @@ -2,8 +2,9 @@ use crate::planner_scenario::_helpers::values::error; use crate::planner_scenario::_helpers::values::string; -use crate::planner_scenario::constants::SCENARIO_PLAN_MISSING_RELATION_TARGET; -use crate::planner_scenario::constants::SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER; +use crate::planner_scenario::constants::{ + SCENARIO_PLAN_MISSING_RELATION_TARGET, SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, +}; use crate::planner_scenario::models::SqlContext; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyStringMethods}; @@ -145,10 +146,3 @@ pub(crate) fn resolve_scenario_check_sql( }; resolve_sql(&context, &args.value("sql")?.extract::()?, false) } - -/// `resolve_scenario_check_sql` as a function of `module`. -pub(crate) fn resolve_scenario_check_sql_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(resolve_scenario_check_sql, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_cli_scenario_plan_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_cli_scenario_plan_function.rs index 7d35e0fb95..6693c30336 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_cli_scenario_plan_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_cli_scenario_plan_function.rs @@ -1,11 +1,10 @@ //! `build_cli_scenario_plan` as a function of `module`. -use crate::planner_scenario::_helpers::cli; +use crate::planner_scenario::_helpers::cli::build_cli_scenario_plan; /// `build_cli_scenario_plan` as a function of `module`. -#[inline] pub fn build_cli_scenario_plan_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - cli::build_cli_scenario_plan_function(module) + pyo3::wrap_pyfunction!(build_cli_scenario_plan, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_artifact_name_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_artifact_name_function.rs index bddf06c54b..539103a224 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_artifact_name_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_artifact_name_function.rs @@ -1,11 +1,10 @@ //! `build_scenario_artifact_name` as a function of `module`. -use crate::planner_scenario::_helpers::artifacts; +use crate::planner_scenario::_helpers::artifacts::build_scenario_artifact_name; /// `build_scenario_artifact_name` as a function of `module`. -#[inline] pub fn build_scenario_artifact_name_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - artifacts::build_scenario_artifact_name_function(module) + pyo3::wrap_pyfunction!(build_scenario_artifact_name, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_execution_plan_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_execution_plan_function.rs index 19ac88867d..2d4b5e7ab6 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_execution_plan_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_execution_plan_function.rs @@ -1,11 +1,10 @@ //! `build_scenario_execution_plan` as a function of `module`. -use crate::planner_scenario::_helpers::execution; +use crate::planner_scenario::_helpers::execution::build_scenario_execution_plan; /// `build_scenario_execution_plan` as a function of `module`. -#[inline] pub fn build_scenario_execution_plan_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - execution::build_scenario_execution_plan_function(module) + pyo3::wrap_pyfunction!(build_scenario_execution_plan, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_fixture_plans_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_fixture_plans_function.rs index ebda2bde0c..251599f757 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_fixture_plans_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_fixture_plans_function.rs @@ -1,11 +1,10 @@ //! `build_scenario_fixture_plans` as a function of `module`. -use crate::planner_scenario::_helpers::fixtures; +use crate::planner_scenario::_helpers::fixtures::build_scenario_fixture_plans; /// `build_scenario_fixture_plans` as a function of `module`. -#[inline] pub fn build_scenario_fixture_plans_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - fixtures::build_scenario_fixture_plans_function(module) + pyo3::wrap_pyfunction!(build_scenario_fixture_plans, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_hash_index_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_hash_index_function.rs index f2fd71baae..cd07e2c3a8 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_hash_index_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_hash_index_function.rs @@ -1,11 +1,10 @@ //! `build_scenario_hash_index` as a function of `module`. -use crate::planner_scenario::_helpers::artifacts; +use crate::planner_scenario::_helpers::artifacts::build_scenario_hash_index; /// `build_scenario_hash_index` as a function of `module`. -#[inline] pub fn build_scenario_hash_index_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - artifacts::build_scenario_hash_index_function(module) + pyo3::wrap_pyfunction!(build_scenario_hash_index, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_map_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_map_function.rs index efa0cc0e7d..4407ecf69e 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_map_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_map_function.rs @@ -1,11 +1,10 @@ //! `build_scenario_relation_map` as a function of `module`. -use crate::planner_scenario::_helpers::artifacts; +use crate::planner_scenario::_helpers::artifacts::build_scenario_relation_map; /// `build_scenario_relation_map` as a function of `module`. -#[inline] pub fn build_scenario_relation_map_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - artifacts::build_scenario_relation_map_function(module) + pyo3::wrap_pyfunction!(build_scenario_relation_map, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_plan_of_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_plan_of_function.rs index a1e335da76..463a332098 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_plan_of_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_plan_of_function.rs @@ -1,11 +1,10 @@ //! `build_scenario_relation_plan_of` as a function of `module`. -use crate::planner_scenario::_helpers::relations; +use crate::planner_scenario::_helpers::relations::build_scenario_relation_plan_of; /// `build_scenario_relation_plan_of` as a function of `module`. -#[inline] pub fn build_scenario_relation_plan_of_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - relations::build_scenario_relation_plan_of_function(module) + pyo3::wrap_pyfunction!(build_scenario_relation_plan_of, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/compute_scenario_hash_prefix_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/compute_scenario_hash_prefix_function.rs index c27bc1c187..559bfcf7ee 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/main/compute_scenario_hash_prefix_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/compute_scenario_hash_prefix_function.rs @@ -1,11 +1,10 @@ //! `compute_scenario_hash_prefix` as a function of `module`. -use crate::planner_scenario::_helpers::artifacts; +use crate::planner_scenario::_helpers::artifacts::compute_scenario_hash_prefix; /// `compute_scenario_hash_prefix` as a function of `module`. -#[inline] pub fn compute_scenario_hash_prefix_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - artifacts::compute_scenario_hash_prefix_function(module) + pyo3::wrap_pyfunction!(compute_scenario_hash_prefix, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/is_scenario_artifact_physical_name.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/is_scenario_artifact_physical_name.rs index c683c80ad9..6ad7d5896c 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/main/is_scenario_artifact_physical_name.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/is_scenario_artifact_physical_name.rs @@ -1,9 +1,9 @@ //! Is scenario artifact physical name. -use crate::planner_scenario::_helpers::artifacts; +use pyo3::prelude::pyfunction; +use sqlbuild_analysis::planner_scenario::main::parse_artifact_name; -/// Is scenario artifact physical name. -#[inline] +#[pyfunction] pub fn is_scenario_artifact_physical_name(name: &str) -> bool { - artifacts::is_scenario_artifact_physical_name(name) + parse_artifact_name::parse_artifact_name(name).is_some() } diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/parse_scenario_artifact_physical_name_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/parse_scenario_artifact_physical_name_function.rs index 0267a2a3a8..97c7b22b55 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/main/parse_scenario_artifact_physical_name_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/parse_scenario_artifact_physical_name_function.rs @@ -1,11 +1,10 @@ //! `parse_scenario_artifact_physical_name` as a function of `module`. -use crate::planner_scenario::_helpers::artifacts; +use crate::planner_scenario::_helpers::artifacts::parse_scenario_artifact_physical_name; /// `parse_scenario_artifact_physical_name` as a function of `module`. -#[inline] pub fn parse_scenario_artifact_physical_name_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - artifacts::parse_scenario_artifact_physical_name_function(module) + pyo3::wrap_pyfunction!(parse_scenario_artifact_physical_name, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph.rs index c2fcd07eda..cc0a8feb14 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph.rs @@ -1,9 +1,7 @@ //! Plan scenario graph. use crate::planner_scenario::_helpers::graph; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyTuple; /// Plan scenario graph. diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph_function.rs index 40d1bfbecb..1d4687a6b6 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph_function.rs @@ -1,11 +1,10 @@ //! `plan_scenario_graph` as a function of `module`. -use crate::planner_scenario::_helpers::graph; +use crate::planner_scenario::_helpers::graph::plan_scenario_graph; /// `plan_scenario_graph` as a function of `module`. -#[inline] pub fn plan_scenario_graph_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - graph::plan_scenario_graph_function(module) + pyo3::wrap_pyfunction!(plan_scenario_graph, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/resolve_scenario_check_sql_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/resolve_scenario_check_sql_function.rs index f3301813d6..5ad1ba6296 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/main/resolve_scenario_check_sql_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/resolve_scenario_check_sql_function.rs @@ -1,11 +1,10 @@ //! `resolve_scenario_check_sql` as a function of `module`. -use crate::planner_scenario::_helpers::sql; +use crate::planner_scenario::_helpers::sql::resolve_scenario_check_sql; /// `resolve_scenario_check_sql` as a function of `module`. -#[inline] pub fn resolve_scenario_check_sql_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - sql::resolve_scenario_check_sql_function(module) + pyo3::wrap_pyfunction!(resolve_scenario_check_sql, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/scenario_artifact_physical_name_check_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/scenario_artifact_physical_name_check_function.rs index 9b75fff57c..1a76d55c8c 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/main/scenario_artifact_physical_name_check_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/scenario_artifact_physical_name_check_function.rs @@ -1,11 +1,10 @@ //! `is_scenario_artifact_physical_name` as a function of `module`. -use crate::planner_scenario::_helpers::artifacts; +use crate::planner_scenario::main::is_scenario_artifact_physical_name::is_scenario_artifact_physical_name; /// `is_scenario_artifact_physical_name` as a function of `module`. -#[inline] pub fn scenario_artifact_physical_name_check_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - artifacts::scenario_artifact_physical_name_check_function(module) + pyo3::wrap_pyfunction!(is_scenario_artifact_physical_name, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/models.rs b/crates/sqlbuild-python-planner/src/planner_scenario/models.rs index d9e42cc382..b385cc1b2f 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/models.rs @@ -1,7 +1,6 @@ //! Public records of planner scenario. -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; +use pyo3::prelude::{Bound, PyAny}; #[derive(Debug)] pub(crate) struct SqlContext<'py> { diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs index a2bd8bcdd4..f94e6c0176 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs @@ -1,31 +1,25 @@ //! Assemble audit plan dataclasses using native rendering and scheduling. -use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::models::GraphIndexes; -use std::collections::HashSet; -use crate::planner_strategy::_helpers::audits::attachment; -use crate::planner_strategy::_helpers::audits::outside_target_lineage; -use crate::planner_strategy::_helpers::audits::resolve_effective_run_scope; +use crate::planner_strategy::_helpers::audits::{ + attachment, outside_target_lineage, resolve_effective_run_scope, +}; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PyString}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_front::attachments::main::assert_no_unresolved_sql_markers::assert_no_unresolved_sql_markers; use sqlbuild_python_project::auditing::main::render_audit_sql::render_audit_sql; -use sqlbuild_python_project::compiled_views::main::project::audits::audits; -use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::graph::models::NativeProjectGraph; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_scope::main::edges::edges; -use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use sqlbuild_python_project::resource_views::main::audit_reads::always_run::always_run; use sqlbuild_python_project::resource_views::main::audit_reads::attached_column_name::attached_column_name; use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_kind::attached_target_kind; -use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_name::attached_target_name; use sqlbuild_python_project::resource_views::main::audit_reads::definition_name::definition_name; use sqlbuild_python_project::resource_views::main::audit_reads::description::description; use sqlbuild_python_project::resource_views::main::audit_reads::evaluation_mode::evaluation_mode; @@ -122,7 +116,7 @@ pub(crate) fn plan_audit_of<'py>( } /// One audit's plan entry, rendered against the planning relations. -fn plan_audit<'py>( +pub(crate) fn plan_audit<'py>( audit: &Bound<'py, PyAny>, request: &AuditPlanning<'py>, ) -> PyResult> { @@ -228,68 +222,3 @@ fn plan_audit<'py>( } new_record(py, RecordId::AuditPlanEntry, (), Some(&values)) } - -/// Select attached audits by their logical target and singular audits by dependency overlap. -pub(crate) fn audit_is_selected( - audit: &Bound<'_, PyAny>, - selected_keys: &HashSet, -) -> PyResult { - let py = audit.py(); - let kind = attached_target_kind(audit)?; - let name = attached_target_name(audit)?; - if !kind.is_none() && !name.is_none() { - let target_value = native_field( - &imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind,))?, - "value", - )?; - let key = (target_value.extract()?, name.extract()?); - return Ok(selected_keys.contains(&key)); - } - for dependency in scope_deps(audit)?.try_iter()? { - if selected_keys.contains(&native_key(&dependency?)?) { - return Ok(true); - } - } - Ok(false) -} - -/// Assemble selected audits in compiled-project order without querying lineage for an empty selection. -pub(crate) fn build_selected_audit_entries<'py>( - request: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let request = Request::new("build_selected_audit_entries", request)?; - let py = request.py(); - let project = request.get("project")?; - let scope = scope_data(&request.get("scope")?)?; - let mut selected: Vec> = Vec::new(); - for audit in audits(&project)?.try_iter()? { - let audit = audit?; - if audit_is_selected(&audit, &scope.selected)? { - selected.push(audit); - } - } - let entries = PyList::empty(py); - if selected.is_empty() { - return Ok(entries); - } - let relations = request.get("relations")?; - let planning = AuditPlanning { - model_locations: native_field(&relations, "model_locations")?, - seed_locations: native_field(&relations, "seed_locations")?, - source_map: native_field(&relations, "source_read_map")?, - adapter: request.get("adapter")?, - model_materializations: request.get("model_materializations")?, - lineage: native_field(&project, "lineage_graph")?.extract()?, - }; - for audit in selected { - entries.append(plan_audit(&audit, &planning)?)?; - } - Ok(entries) -} - -/// `plan_audit_of` as a function of `module`. -pub(crate) fn plan_audit_of_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(plan_audit_of, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs index af0f930f91..d393d68ca8 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs @@ -1,11 +1,13 @@ //! Audit attachment, target lineage and run scope. +use crate::planner_strategy::main::values::input_error::input_error; +use crate::planner_strategy::main::values::text::text; use std::cell::RefCell; use std::collections::HashMap; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::main::reachable_graph_keys::reachable_graph_keys; @@ -15,9 +17,7 @@ use sqlbuild_analysis::planner_strategy::main::effective_run_scope::effective_ru use sqlbuild_analysis::planner_strategy::main::singular_attachment::singular_attachment; use sqlbuild_analysis::planner_strategy::models::{AuditAttachmentKind, AuditRunScope}; -use crate::planner_strategy::_helpers::values::input_error; use crate::planner_strategy::_helpers::values::optional_string; -use crate::planner_strategy::_helpers::values::text; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; @@ -187,9 +187,3 @@ pub(crate) fn resolve_effective_run_scope<'py>( }; imported(py, AUDITING_TYPES_MODULE, "AuditRunScope")?.call1((scope,)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_attachment_kind, module)?)?; - module.add_function(wrap_pyfunction!(reads_outside_target_lineage, module)?)?; - module.add_function(wrap_pyfunction!(resolve_effective_run_scope, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_inputs.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_inputs.rs index 7945307cc4..24b958f4ef 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_inputs.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_inputs.rs @@ -1,5 +1,8 @@ //! Cursor input relations and authored cursor input column checks of model plan entries. +use crate::planner_strategy::main::lineage::resolve_lineage_reference::resolve_lineage_reference; +use crate::planner_strategy::main::values::input_error::input_error; +use crate::planner_strategy::main::values::text::text; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyFrozenSet, PyString, PyTuple}; @@ -8,9 +11,6 @@ use sqlbuild_analysis::planner_strategy::main::cursor_input_column::cursor_input use sqlbuild_analysis::planner_strategy::models::{CursorInputColumnCheck, CursorInputColumnOwner}; use crate::planner_identity::main::or_empty::or_empty; -use crate::planner_strategy::_helpers::lineage::resolve_lineage_reference; -use crate::planner_strategy::_helpers::values::input_error; -use crate::planner_strategy::_helpers::values::text; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -25,7 +25,6 @@ use sqlbuild_python_front::model_config::main::config::config_text_object::confi use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; -use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use sqlbuild_python_project::planner_strategy::constants::{ ENFORCED_CONTRACT, INCREMENTAL_MATERIALIZATION, MODEL_REFERENCE, SEED_REFERENCE, SOURCE_REFERENCE, TIMESTAMP_CURSOR, @@ -48,7 +47,7 @@ fn incremental_cursor_flag( } /// The names of a sequence of objects with a `name`. -fn names(items: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn names(items: &Bound<'_, PyAny>) -> PyResult> { items .try_iter()? .map(|item| text(&native_field(&item?, "name")?)) @@ -334,49 +333,3 @@ pub(crate) fn build_cursor_input_relations<'py>( } PyTuple::new(py, relations) } - -/// Selected models and seeds, and selected sources with loaders, whose relations this invocation produces. -#[pyfunction] -#[pyo3(signature = (*, scope, source_map))] -pub(crate) fn runtime_cursor_producer_names<'py>( - scope: &Bound<'py, PyAny>, - source_map: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = scope.py(); - let data = scope_data(scope)?; - let mut names: Vec<&str> = Vec::new(); - for (kind, name) in &data.selected { - match kind.as_str() { - "model" | "seed" => names.push(name), - "source" => { - let source: Bound<'py, PyAny> = mapping_get(source_map, name)?; - if !source.is_none() && !native_field(&source, "loader")?.is_none() { - names.push(name); - } - } - _ => {} - } - } - PyFrozenSet::new(py, names) -} - -/// `cursor_input_columns_of` as a function of `module`. -pub(crate) fn cursor_input_columns_of_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(cursor_input_columns_of, module) -} - -/// `cursor_input_relations_of` as a function of `module`. -pub(crate) fn cursor_input_relations_of_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(cursor_input_relations_of, module) -} - -/// `runtime_cursor_producer_names` as a function of `module`. -pub(crate) fn runtime_cursor_producer_names_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(runtime_cursor_producer_names, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs index 986b260bfd..3fa759c175 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs @@ -1,30 +1,26 @@ //! Warehouse cursor column type checks. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::planner_strategy::main::values::text::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_strategy::errors::TypeParseFailure; -use sqlbuild_analysis::planner_strategy::main::classify_cursor_sql_type::classify_cursor_sql_type as classify; use sqlbuild_analysis::planner_strategy::main::cursor_type_warning::cursor_type_warning; -use sqlbuild_analysis::planner_strategy::models::{CursorType, CursorTypeCheck}; +use sqlbuild_analysis::planner_strategy::models::CursorTypeCheck; use sqlbuild_host::entries::models::ExecutorHost; -use crate::planner_changes::main::planner_enum::planner_enum; -use crate::planner_strategy::_helpers::values::optional_string; -use crate::planner_strategy::_helpers::values::plan_warning; -use crate::planner_strategy::_helpers::values::text; +use crate::planner_strategy::_helpers::values::{optional_string, plan_warning}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; const PLANNER_LOGGER: &str = "sqlbuild.planner"; /// Log the Polyglot parse error that left a warehouse type unclassified. -fn log_parse_failure(py: Python<'_>, failure: &TypeParseFailure) -> PyResult<()> { +pub(crate) fn log_parse_failure(py: Python<'_>, failure: &TypeParseFailure) -> PyResult<()> { let text = |value: &str| PyString::new(py, value).into_any(); call_with( &host_entry(py, ExecutorHost::LogDebugEvent)?, @@ -44,33 +40,10 @@ fn log_parse_failure(py: Python<'_>, failure: &TypeParseFailure) -> PyResult<()> .map(|_| ()) } -/// The `CursorType` a supported cross-adapter SQL type holds, or `None`. -pub(crate) fn classify_cursor_sql_type<'py>( - py: Python<'py>, - sql_type: &str, -) -> PyResult>> { - let detected: Option = match classify(sql_type) { - Ok(detected) => detected, - Err(failure) => { - log_parse_failure(py, &failure)?; - None - } - }; - detected - .map(|cursor_type| { - let value: &str = match cursor_type { - CursorType::Timestamp => "timestamp", - CursorType::Integer => "integer", - }; - planner_enum(py, RecordId::CursorType, value) - }) - .transpose() -} - /// A `PlanWarning` when the warehouse cursor column type contradicts `cursor_type`. #[pyfunction] #[pyo3(signature = (**options))] -fn check_cursor_type_consistency<'py>( +pub(crate) fn check_cursor_type_consistency<'py>( py: Python<'py>, options: Option>, ) -> PyResult>> { @@ -126,7 +99,3 @@ pub(crate) fn cursor_type_warning_record<'py>( } warning.map(|row| plan_warning(model_name, row)).transpose() } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(check_cursor_type_consistency, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs index 4126c3d166..d81b0868fe 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs @@ -1,18 +1,14 @@ //! Watermark inputs resolved through SQL reference lineage, and capped producers behind them. +use crate::planner_strategy::main::values::input_error::input_error; +use crate::planner_strategy::main::values::text::text; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::PyDict; use sqlbuild_analysis::planner_strategy::errors::LineageFailure; -use sqlbuild_analysis::planner_strategy::main::capped_watermark_input::capped_watermark_input; -use sqlbuild_analysis::planner_strategy::main::lineage_reference::lineage_reference; use sqlbuild_analysis::planner_strategy::models::{ProducerFacts, ReferenceIdentity}; use sqlbuild_analysis::planner_strategy::types::ReferenceGraph; -use crate::planner_strategy::_helpers::microbatches::CAP_ACTIONS; -use crate::planner_strategy::_helpers::microbatches::microbatch_limit_config; -use crate::planner_strategy::_helpers::values::input_error; -use crate::planner_strategy::_helpers::values::text; +use crate::planner_strategy::_helpers::microbatches::{CAP_ACTIONS, microbatch_limit_config}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; @@ -20,15 +16,14 @@ use sqlbuild_python_front::model_config::main::config::config_text_is::config_te use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; use sqlbuild_python_project::compiled_views::main::compiled_models::config::config; -use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; use sqlbuild_python_project::native_host::models::ModelLookup; const MICROBATCH_MODE: &str = "microbatch"; /// The project's models and SQL functions by name. -struct ProjectLineage<'py> { - models_by_name: ModelLookup, - functions_by_name: Bound<'py, PyAny>, +pub(crate) struct ProjectLineage<'py> { + pub(crate) models_by_name: ModelLookup, + pub(crate) functions_by_name: Bound<'py, PyAny>, } /// The references of the value under `name`, when the mapping has it. @@ -43,14 +38,14 @@ fn mapped_references<'py>( references(&value).map(Some) } -fn references<'py>(owner: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn references<'py>(owner: &Bound<'py, PyAny>) -> PyResult>> { sqlbuild_python_project::resource_views::main::resource_reads::references::references(owner)? .try_iter()? .collect() } /// Whether a model's `incremental_mode` is microbatch. -fn microbatch_flag(model: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn microbatch_flag(model: &Bound<'_, PyAny>) -> PyResult { config_text_is( model.py(), &model_values(model)?, @@ -60,7 +55,7 @@ fn microbatch_flag(model: &Bound<'_, PyAny>) -> PyResult { } /// A model's watermark input names. -fn watermark_names(model: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn watermark_names(model: &Bound<'_, PyAny>) -> PyResult> { native_field(&resolve_cursor_input_roles(model)?, "watermark_inputs")? .try_iter()? .map(|name| text(&name?)) @@ -109,7 +104,7 @@ impl<'py> ReferenceGraph for ProjectLineage<'py> { } /// The Python error of a failed lineage walk. -fn lineage_error(py: Python<'_>, failure: LineageFailure) -> PyErr { +pub(crate) fn lineage_error(py: Python<'_>, failure: LineageFailure) -> PyErr { match failure { LineageFailure::Graph(error) => error, LineageFailure::NotInLineage { model, input } => input_error( @@ -120,52 +115,3 @@ fn lineage_error(py: Python<'_>, failure: LineageFailure) -> PyErr { ), } } - -/// The reference one named input resolves to through model and SQL function references. -pub(crate) fn resolve_lineage_reference<'py>( - model: &Bound<'py, PyAny>, - input_name: &str, - models_by_name: ModelLookup, - functions_by_name: Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = model.py(); - let graph = ProjectLineage { - models_by_name, - functions_by_name, - }; - let name: String = model_name(model)?; - lineage_reference(&graph, (&name, references(model)?), input_name) - .map_err(|failure| lineage_error(py, failure)) -} - -/// Reject capped producers feeding a microbatch model's watermark inputs. -pub(crate) fn reject_capped_watermark_inputs<'py>( - model: &Bound<'py, PyAny>, - models_by_name: ModelLookup, - functions_by_name: Option>, -) -> PyResult<()> { - let py: Python<'py> = model.py(); - if !microbatch_flag(model)? { - return Ok(()); - } - let functions_by_name: Bound<'py, PyAny> = match functions_by_name { - Some(functions) if functions.is_truthy()? => functions, - _ => PyDict::new(py).into_any(), - }; - let graph = ProjectLineage { - models_by_name, - functions_by_name, - }; - let name: String = model_name(model)?; - let inputs: Vec = watermark_names(model)?; - match capped_watermark_input(&graph, (&name, references(model)?), inputs) { - Ok(None) => Ok(()), - Ok(Some(capped)) => Err(input_error( - py, - ErrorId::CompileInput, - &capped.message(), - None, - )), - Err(failure) => Err(lineage_error(py, failure)), - } -} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs index 06cab7ab70..a51ea10ee2 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs @@ -1,10 +1,11 @@ //! Microbatch limits applied to planned model entries in CLI, project and model precedence. use crate::planner_changes::main::native_enum::native_enum; +use crate::planner_strategy::main::values::input_error::input_error; +use crate::planner_strategy::main::values::text::text; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction}; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; @@ -15,12 +16,10 @@ use sqlbuild_core::plan_display::main::effective_timestamp_grain::effective_time use sqlbuild_core::plan_display::models::TimestampGrainFacts; use crate::plan_output::main::resolve_effective_microbatch_batch_size::resolve_effective_microbatch_batch_size; -use crate::planner_strategy::_helpers::microbatches::CAP_ACTIONS; -use crate::planner_strategy::_helpers::microbatches::capped_microbatch_bounds; -use crate::planner_strategy::_helpers::microbatches::count_microbatches; -use crate::planner_strategy::_helpers::values::input_error; +use crate::planner_strategy::_helpers::microbatches::{ + CAP_ACTIONS, capped_microbatch_bounds, count_microbatches, +}; use crate::planner_strategy::_helpers::values::optional_string; -use crate::planner_strategy::_helpers::values::text; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; @@ -93,7 +92,7 @@ fn batch_count(entry: &Bound<'_, PyAny>) -> PyResult> { } /// The entry with one batch limit applied, and the warning when a non-error limit is exceeded. -fn limited<'py>( +pub(crate) fn limited<'py>( entry: &Bound<'py, PyAny>, max_batches: Bound<'py, PyAny>, action: Bound<'py, PyAny>, @@ -194,7 +193,7 @@ pub(crate) fn apply_microbatch_limit<'py>( } /// `(entry, warnings)` with the limit outcome's warning, if any. -fn with_warnings<'py>( +pub(crate) fn with_warnings<'py>( (entry, warning): LimitedEntry<'py>, mut warnings: Vec>, ) -> PyResult> { @@ -204,7 +203,7 @@ fn with_warnings<'py>( } /// The default limit: none for an incremental rolling window, else the model's limit or the configured ceiling. -fn default_limit<'py>( +pub(crate) fn default_limit<'py>( entry: &Bound<'py, PyAny>, configured: Bound<'py, PyAny>, ) -> PyResult> { @@ -223,53 +222,3 @@ fn default_limit<'py>( configured }) } - -/// Apply CLI, project, and model batch limits in their documented precedence order. -#[pyfunction] -#[pyo3(signature = (*, entry, inputs))] -pub(crate) fn apply_configured_microbatch_limits<'py>( - entry: &Bound<'py, PyAny>, - inputs: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = entry.py(); - let configured: Bound<'py, PyAny> = native_field(inputs, "max_microbatches")?; - let configured_action: Bound<'py, PyAny> = native_field(inputs, "microbatch_limit_action")?; - if native_field(inputs, "max_microbatches_is_override")?.is_truthy()? { - let error: Bound<'py, PyAny> = enum_member(py, RecordId::MicrobatchLimitAction, "ERROR")?; - return with_warnings(limited(entry, configured, error)?, Vec::new()); - } - let declared: Bound<'py, PyAny> = native_field(entry, "declared_microbatch_limit_action")?; - if declared.is_none() { - let maximum: Bound<'py, PyAny> = default_limit(entry, configured)?; - return with_warnings(limited(entry, maximum, configured_action)?, Vec::new()); - } - let mut warnings: Vec> = Vec::new(); - let mut entry: Bound<'py, PyAny> = entry.clone(); - if !configured.is_none() { - let (_, safety_warning) = limited(&entry, configured.clone(), configured_action.clone())?; - entry = replace( - &entry, - &[ - ("microbatch_safety_limit", configured), - ("microbatch_safety_limit_action", configured_action), - ], - )?; - warnings.extend(safety_warning); - } - let model_limit: Bound<'py, PyAny> = native_field(&entry, "microbatch_limit")?; - with_warnings(limited(&entry, model_limit, declared)?, warnings) -} - -/// `apply_configured_microbatch_limits` as a function of `module`. -pub(crate) fn apply_configured_microbatch_limits_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(apply_configured_microbatch_limits, module) -} - -/// `apply_microbatch_limit` as a function of `module`. -pub(crate) fn apply_microbatch_limit_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(apply_microbatch_limit, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatches.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatches.rs index 19457fb1ab..3a5482af5b 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatches.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatches.rs @@ -1,12 +1,11 @@ //! Microbatch counts, capped ranges and batch-limit configuration. +use crate::planner_strategy::main::values::text::text; use std::marker::PhantomData; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyString, PyTuple, PyTupleMethods}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::planner_strategy::main::cap_integer_bounds::cap_integer_bounds; use sqlbuild_analysis::planner_strategy::main::cap_timeline_bounds::cap_timeline_bounds; @@ -19,7 +18,6 @@ use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; use sqlbuild_cursor::algebra::models::{DurationDigits, DurationSpan}; use crate::plan_output::main::aligned_timestamp_bounds::aligned_timestamp_bounds; -use crate::planner_strategy::_helpers::values::text; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; @@ -236,7 +234,7 @@ pub(crate) fn capped_microbatch_bounds<'py>( /// The prominent warning or error message when one model exceeds its batch limit. #[pyfunction] #[pyo3(signature = (*, model_name, max_batches, batch_count, action))] -fn microbatch_limit_warning( +pub(crate) fn microbatch_limit_warning( model_name: &str, max_batches: Option, batch_count: i128, @@ -283,8 +281,3 @@ pub(crate) fn microbatch_limit_config<'py>( let legacy: Bound<'py, PyAny> = mapping_get(values, "max_microbatches")?; Ok((positive_integer(Some(legacy))?, None)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(microbatch_limit_warning, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs index ef946bd1da..bc5e45b43f 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs @@ -1,9 +1,11 @@ //! Assemble model plan dataclasses from resolved plan decisions and authored model config. +use crate::planner_strategy::main::values::input_error::input_error; +use crate::planner_strategy::main::values::text::text; +use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::planner_strategy::main::group_replacement_refusal::group_replacement_refusal; use sqlbuild_analysis::planner_strategy::main::materialization_type::materialization_type; @@ -17,18 +19,15 @@ use sqlbuild_model_config::execution_limits::main::parse_execution_limit::parse_ use sqlbuild_model_config::execution_limits::models::ExecutionLimit; use sqlbuild_model_config::model_validation::constants::PHYSICAL_STORAGE_KEYS; -use crate::planner_strategy::_helpers::cursor_types::cursor_type_warning_record; -use crate::planner_strategy::_helpers::cursor_types::warehouse_column_types; -use crate::planner_strategy::_helpers::microbatch_limits::replay_grain; -use crate::planner_strategy::_helpers::microbatch_limits::unknown_grain; +use crate::planner_strategy::_helpers::cursor_types::{ + cursor_type_warning_record, warehouse_column_types, +}; +use crate::planner_strategy::_helpers::microbatch_limits::{replay_grain, unknown_grain}; use crate::planner_strategy::_helpers::microbatches::microbatch_limit_config; -use crate::planner_strategy::_helpers::strategy::model_warning_records; -use crate::planner_strategy::_helpers::strategy::resolve_schema_actions; -use crate::planner_strategy::_helpers::strategy::warning_inputs; -use crate::planner_strategy::_helpers::values::input_error; -use crate::planner_strategy::_helpers::values::optional_string; -use crate::planner_strategy::_helpers::values::plan_warning; -use crate::planner_strategy::_helpers::values::text; +use crate::planner_strategy::_helpers::strategy::{ + model_warning_records, resolve_schema_actions, warning_inputs, +}; +use crate::planner_strategy::_helpers::values::{optional_string, plan_warning}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; @@ -113,7 +112,7 @@ fn column_info<'py>( } /// Declared schema columns as `ColumnInfo`: typed columns only, or every column with an empty default type. -fn schema_columns<'py>( +pub(crate) fn schema_columns<'py>( model: &Bound<'py, PyAny>, typed_only: bool, ) -> PyResult> { @@ -228,7 +227,7 @@ fn type_enforcement<'py>(model: &Bound<'py, PyAny>) -> PyResult, effective_grain: Option, ) -> PyResult> { @@ -250,21 +249,6 @@ fn lookback_of( microbatch_lookback(&config).map_err(|error| unknown_grain(model.py(), error)) } -/// Resolve explicit or default lookback for one microbatch model. -#[pyfunction] -#[pyo3(signature = (*, model, effective_grain = None))] -pub(crate) fn resolve_microbatch_lookback( - model: &Bound<'_, PyAny>, - effective_grain: Option, -) -> PyResult> { - lookback_of(model, effective_grain) -} - -/// Declared seed columns with types, as `ColumnInfo`. -pub(crate) fn extract_seed_columns<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - schema_columns(seed, true) -} - /// The adapter-rendered logical DDL of one plan action. fn logical_ddl<'py>( request: &ModelEntryDecisions<'py>, @@ -784,10 +768,3 @@ pub(crate) fn assemble_model_plan_entry<'py>( let plan_entry: Bound<'py, PyAny> = new_record(py, RecordId::ModelPlanEntry, (), Some(&entry))?; Ok((plan_entry, warnings)) } - -/// `resolve_microbatch_lookback` as a function of `module`. -pub(crate) fn resolve_microbatch_lookback_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(resolve_microbatch_lookback, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs index 96aebe881f..fe225e30fe 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs @@ -1,34 +1,24 @@ //! Per-model plan actions, schema actions and plan warnings. use crate::planner_changes::main::native_enum::native_enum; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use crate::planner_strategy::main::values::text::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_analysis::planner_strategy::main::materialization_type::materialization_type; -use sqlbuild_analysis::planner_strategy::main::model_plan_action::model_plan_action; use sqlbuild_analysis::planner_strategy::main::model_warnings::model_warnings; use sqlbuild_analysis::planner_strategy::main::schema_actions::schema_actions; use sqlbuild_analysis::planner_strategy::models::{ - ModelStrategy, PlanAction, PlanReason, SchemaActionKind, WarningInputs, + PlanAction, PlanReason, SchemaActionKind, WarningInputs, }; -use crate::planner_changes::main::planner_enum::planner_enum; -use crate::planner_strategy::_helpers::values::change_facts; -use crate::planner_strategy::_helpers::values::config_value; -use crate::planner_strategy::_helpers::values::finding_rows; -use crate::planner_strategy::_helpers::values::input_error; -use crate::planner_strategy::_helpers::values::materialization_value; -use crate::planner_strategy::_helpers::values::model_materialization; -use crate::planner_strategy::_helpers::values::optional_string; -use crate::planner_strategy::_helpers::values::plan_warning; -use crate::planner_strategy::_helpers::values::text; +use crate::planner_strategy::_helpers::values::{change_facts, finding_rows, plan_warning}; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -fn action_value(action: PlanAction) -> &'static str { +pub(crate) fn action_value(action: PlanAction) -> &'static str { match action { PlanAction::CreateView => "create_view", PlanAction::CreateTable => "create_table", @@ -41,7 +31,7 @@ fn action_value(action: PlanAction) -> &'static str { } } -fn reason_value(reason: PlanReason) -> &'static str { +pub(crate) fn reason_value(reason: PlanReason) -> &'static str { match reason { PlanReason::FirstRun => "first_run", PlanReason::Renamed => "renamed", @@ -64,56 +54,6 @@ fn schema_policy(on_schema_change: &Bound<'_, PyAny>) -> PyResult text(on_schema_change).map(Some) } -/// The `MaterializationType` of a model's `materialized` config: table when unset, custom when unknown. -#[pyfunction(name = "get_materialization_type")] -#[pyo3(signature = (model))] -pub(crate) fn materialization_of<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - planner_enum( - model.py(), - RecordId::MaterializationType, - materialization_value(model_materialization(model)?), - ) -} - -/// The plan action and reason for one model. -#[pyfunction] -#[pyo3(signature = (*, model, change_result, full_refresh))] -pub(crate) fn resolve_model_plan_action<'py>( - model: &Bound<'py, PyAny>, - change_result: &Bound<'py, PyAny>, - full_refresh: bool, -) -> PyResult> { - let py: Python<'py> = model.py(); - let enabled: Bound<'py, PyAny> = config_value(model, "enabled")?; - let strategy = ModelStrategy { - name: - sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name( - model, - )?, - disabled: enabled.is_instance_of::() && !enabled.is_truthy()?, - materialization: model_materialization(model)?, - incremental_strategy: optional_string(&config_value(model, "incremental_strategy")?)?, - }; - let (action, reason) = - model_plan_action(&strategy, &change_facts(change_result)?, full_refresh).map_err( - |error| { - input_error( - py, - ErrorId::PlannerInput, - &error.message(), - Some(error.code()), - ) - }, - )?; - PyTuple::new( - py, - [ - planner_enum(py, RecordId::PlanAction, action_value(action))?, - planner_enum(py, RecordId::PlanReason, reason_value(reason))?, - ], - ) -} - /// Concrete schema change actions from findings and `on_schema_change`. #[pyfunction] #[pyo3(signature = (*, schema_findings, on_schema_change))] @@ -180,7 +120,7 @@ pub(crate) fn model_warning_records<'py>( /// Warnings for one model from its change detection and plan decisions. #[pyfunction] #[pyo3(signature = (**options))] -fn build_model_warnings<'py>( +pub(crate) fn build_model_warnings<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -215,10 +155,3 @@ fn build_model_warnings<'py>( model_warning_records(&model_name, &inputs, &arguments.value("change_result")?)?, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(materialization_of, module)?)?; - module.add_function(wrap_pyfunction!(resolve_model_plan_action, module)?)?; - module.add_function(wrap_pyfunction!(resolve_schema_actions, module)?)?; - module.add_function(wrap_pyfunction!(build_model_warnings, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs index 4f7a256c1b..d910fd16b3 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs @@ -1,9 +1,10 @@ //! Conversions between the planner's Python models and the native strategy rows. use crate::planner_changes::main::native_enum::native_enum; +use crate::planner_strategy::main::values::text::text; use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_front::model_config::main::config::config_value::config_value as typed_config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; @@ -16,20 +17,13 @@ use sqlbuild_analysis::planner_strategy::models::{ ChangeFacts, MaterializationType, PlanChangeKind, WarningRow, WarningSeverity, }; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; const YML_SOURCE: &str = "yml"; const FULL_BACKFILL: &str = "full"; const FORWARD_ONLY_BACKFILL: &str = "forward"; -/// `str(value)` of a Python string or string enum. -pub(crate) fn text(value: &Bound<'_, PyAny>) -> PyResult { - value.extract() -} - /// A string value, or `None` for `None` and anything that is not a string. pub(crate) fn optional_string(value: &Bound<'_, PyAny>) -> PyResult> { if value.is_instance_of::() { @@ -149,23 +143,3 @@ pub(crate) fn plan_warning<'py>( ], ) } - -/// The native error `class(message, code=code)`. -pub(crate) fn input_error( - py: Python<'_>, - class: ErrorId, - message: &str, - code: Option<&str>, -) -> PyErr { - let raised: PyResult> = error_class(py, class).and_then(|class| { - let options: Bound<'_, PyDict> = PyDict::new(py); - if let Some(code) = code { - options.set_item("code", code)?; - } - class.call((message,), Some(&options)) - }); - match raised { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } -} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs index 9135742077..ab7a81e959 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs @@ -1,17 +1,36 @@ //! Select attached audits by their logical target and singular audits by dependency overlap. -use crate::planner_strategy::_helpers::audit_entries; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::planner_strategy::constants::COMPILE_TYPES_MODULE; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_kind::attached_target_kind; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_name::attached_target_name; +use sqlbuild_python_project::resource_views::main::audit_reads::scope_deps::scope_deps; use std::collections::HashSet; /// Select attached audits by their logical target and singular audits by dependency overlap. -#[inline] pub fn audit_is_selected( audit: &Bound<'_, PyAny>, selected_keys: &HashSet, ) -> PyResult { - audit_entries::audit_is_selected(audit, selected_keys) + let py = audit.py(); + let kind = attached_target_kind(audit)?; + let name = attached_target_name(audit)?; + if !kind.is_none() && !name.is_none() { + let target_value = native_field( + &imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind,))?, + "value", + )?; + let key = (target_value.extract()?, name.extract()?); + return Ok(selected_keys.contains(&key)); + } + for dependency in scope_deps(audit)?.try_iter()? { + if selected_keys.contains(&native_key(&dependency?)?) { + return Ok(true); + } + } + Ok(false) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs index 13a48a73a1..82e778151a 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs @@ -1,15 +1,45 @@ //! Assemble selected audits in compiled-project order without querying lineage for an empty selection. -use crate::planner_strategy::_helpers::audit_entries; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyList; +use crate::planner_strategy::_helpers::audit_entries::plan_audit; +use crate::planner_strategy::main::audit_entries::audit_is_selected::audit_is_selected; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyList, PyListMethods}; +use sqlbuild_python_base::boundary::models::Request; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::project::audits::audits; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_strategy::models::AuditPlanning; /// Assemble selected audits in compiled-project order without querying lineage for an empty selection. -#[inline] pub fn build_selected_audit_entries<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { - audit_entries::build_selected_audit_entries(request) + let request = Request::new("build_selected_audit_entries", request)?; + let py = request.py(); + let project = request.get("project")?; + let scope = scope_data(&request.get("scope")?)?; + let mut selected: Vec> = Vec::new(); + for audit in audits(&project)?.try_iter()? { + let audit = audit?; + if audit_is_selected(&audit, &scope.selected)? { + selected.push(audit); + } + } + let entries = PyList::empty(py); + if selected.is_empty() { + return Ok(entries); + } + let relations = request.get("relations")?; + let planning = AuditPlanning { + model_locations: native_field(&relations, "model_locations")?, + seed_locations: native_field(&relations, "seed_locations")?, + source_map: native_field(&relations, "source_read_map")?, + adapter: request.get("adapter")?, + model_materializations: request.get("model_materializations")?, + lineage: native_field(&project, "lineage_graph")?.extract()?, + }; + for audit in selected { + entries.append(plan_audit(&audit, &planning)?)?; + } + Ok(entries) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/plan_audit_of_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/plan_audit_of_function.rs index 7900703a0c..b09754ca80 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/plan_audit_of_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/plan_audit_of_function.rs @@ -1,11 +1,10 @@ //! `plan_audit_of` as a function of `module`. -use crate::planner_strategy::_helpers::audit_entries; +use crate::planner_strategy::_helpers::audit_entries::plan_audit_of; /// `plan_audit_of` as a function of `module`. -#[inline] pub fn plan_audit_of_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - audit_entries::plan_audit_of_function(module) + pyo3::wrap_pyfunction!(plan_audit_of, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs index 9c28e5c8d6..f1c8160e55 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs @@ -1,12 +1,14 @@ //! Register the audits functions and classes. -use crate::planner_strategy::_helpers::audits; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_strategy::_helpers::audits::reads_outside_target_lineage; +use crate::planner_strategy::_helpers::audits::resolve_attachment_kind; +use crate::planner_strategy::_helpers::audits::resolve_effective_run_scope; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the audits functions and classes. -#[inline] pub fn register_audits(module: &Bound<'_, PyModule>) -> PyResult<()> { - audits::register(module) + module.add_function(wrap_pyfunction!(resolve_attachment_kind, module)?)?; + module.add_function(wrap_pyfunction!(reads_outside_target_lineage, module)?)?; + module.add_function(wrap_pyfunction!(resolve_effective_run_scope, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/build_cursor_input_relations.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/build_cursor_input_relations.rs index a64b2148fa..ea671898f1 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/build_cursor_input_relations.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/build_cursor_input_relations.rs @@ -1,9 +1,7 @@ //! Cursor-bearing input relations of an incremental model, for runtime range discovery. use crate::planner_strategy::_helpers::cursor_inputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyTuple; use sqlbuild_python_project::planner_strategy::models::CursorInputLookups; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_columns_of_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_columns_of_function.rs index 0d30b3e9f6..ec1cf594b8 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_columns_of_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_columns_of_function.rs @@ -1,11 +1,10 @@ //! `cursor_input_columns_of` as a function of `module`. -use crate::planner_strategy::_helpers::cursor_inputs; +use crate::planner_strategy::_helpers::cursor_inputs::cursor_input_columns_of; /// `cursor_input_columns_of` as a function of `module`. -#[inline] pub fn cursor_input_columns_of_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - cursor_inputs::cursor_input_columns_of_function(module) + pyo3::wrap_pyfunction!(cursor_input_columns_of, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_relations_of_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_relations_of_function.rs index 482ac222e8..22a132793f 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_relations_of_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_relations_of_function.rs @@ -1,11 +1,10 @@ //! `cursor_input_relations_of` as a function of `module`. -use crate::planner_strategy::_helpers::cursor_inputs; +use crate::planner_strategy::_helpers::cursor_inputs::cursor_input_relations_of; /// `cursor_input_relations_of` as a function of `module`. -#[inline] pub fn cursor_input_relations_of_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - cursor_inputs::cursor_input_relations_of_function(module) + pyo3::wrap_pyfunction!(cursor_input_relations_of, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/reject_unexposed_cursor_input_columns.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/reject_unexposed_cursor_input_columns.rs index 4c82b8ebce..efb905a3f5 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/reject_unexposed_cursor_input_columns.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/reject_unexposed_cursor_input_columns.rs @@ -1,9 +1,7 @@ //! Reject authored cursor input columns a contract or the source warehouse metadata does not expose. use crate::planner_strategy::_helpers::cursor_inputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_python_project::planner_strategy::models::CursorInputLookups; /// Reject authored cursor input columns a contract or the source warehouse metadata does not expose. diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names.rs index 0f5bdd1309..306e46dcfa 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names.rs @@ -1,16 +1,33 @@ //! Selected models and seeds, and selected sources with loaders, whose relations this invocation produces. -use crate::planner_strategy::_helpers::cursor_inputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyFrozenSet; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; /// Selected models and seeds, and selected sources with loaders, whose relations this invocation produces. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, scope, source_map))] pub fn runtime_cursor_producer_names<'py>( scope: &Bound<'py, PyAny>, source_map: &Bound<'py, PyAny>, ) -> PyResult> { - cursor_inputs::runtime_cursor_producer_names(scope, source_map) + let py: Python<'py> = scope.py(); + let data = scope_data(scope)?; + let mut names: Vec<&str> = Vec::new(); + for (kind, name) in &data.selected { + match kind.as_str() { + "model" | "seed" => names.push(name), + "source" => { + let source: Bound<'py, PyAny> = mapping_get(source_map, name)?; + if !source.is_none() && !native_field(&source, "loader")?.is_none() { + names.push(name); + } + } + _ => {} + } + } + PyFrozenSet::new(py, names) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names_function.rs index 720442a149..03d89f796c 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names_function.rs @@ -1,11 +1,10 @@ //! `runtime_cursor_producer_names` as a function of `module`. -use crate::planner_strategy::_helpers::cursor_inputs; +use crate::planner_strategy::main::cursor_inputs::runtime_cursor_producer_names::runtime_cursor_producer_names; /// `runtime_cursor_producer_names` as a function of `module`. -#[inline] pub fn runtime_cursor_producer_names_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - cursor_inputs::runtime_cursor_producer_names_function(module) + pyo3::wrap_pyfunction!(runtime_cursor_producer_names, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/classify_cursor_sql_type.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/classify_cursor_sql_type.rs index 48408a7f55..5c3153f9ce 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/classify_cursor_sql_type.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/classify_cursor_sql_type.rs @@ -1,16 +1,31 @@ //! The `CursorType` a supported cross-adapter SQL type holds, or `None`. -use crate::planner_strategy::_helpers::cursor_types; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::planner_changes::main::planner_enum::planner_enum; +use crate::planner_strategy::_helpers::cursor_types::log_parse_failure; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_analysis::planner_strategy::main::classify_cursor_sql_type::classify_cursor_sql_type as classify; +use sqlbuild_analysis::planner_strategy::models::CursorType; +use sqlbuild_python_base::records::types::RecordId; /// The `CursorType` a supported cross-adapter SQL type holds, or `None`. -#[inline] pub fn classify_cursor_sql_type<'py>( py: Python<'py>, sql_type: &str, ) -> PyResult>> { - cursor_types::classify_cursor_sql_type(py, sql_type) + let detected: Option = match classify(sql_type) { + Ok(detected) => detected, + Err(failure) => { + log_parse_failure(py, &failure)?; + None + } + }; + detected + .map(|cursor_type| { + let value: &str = match cursor_type { + CursorType::Timestamp => "timestamp", + CursorType::Integer => "integer", + }; + planner_enum(py, RecordId::CursorType, value) + }) + .transpose() } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/register_cursor_types.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/register_cursor_types.rs index 90b4e247a4..5f5507ec6b 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/register_cursor_types.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/register_cursor_types.rs @@ -1,12 +1,10 @@ //! Register the cursor types functions and classes. -use crate::planner_strategy::_helpers::cursor_types; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_strategy::_helpers::cursor_types::check_cursor_type_consistency; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the cursor types functions and classes. -#[inline] pub fn register_cursor_types(module: &Bound<'_, PyModule>) -> PyResult<()> { - cursor_types::register(module) + module.add_function(wrap_pyfunction!(check_cursor_type_consistency, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/reject_capped_watermark_inputs.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/reject_capped_watermark_inputs.rs index bf260c1a71..01372664aa 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/reject_capped_watermark_inputs.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/reject_capped_watermark_inputs.rs @@ -1,17 +1,46 @@ //! Reject capped producers feeding a microbatch model's watermark inputs. -use crate::planner_strategy::_helpers::lineage; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_strategy::_helpers::lineage::ProjectLineage; +use crate::planner_strategy::_helpers::lineage::lineage_error; +use crate::planner_strategy::_helpers::lineage::microbatch_flag; +use crate::planner_strategy::_helpers::lineage::references; +use crate::planner_strategy::_helpers::lineage::watermark_names; +use crate::planner_strategy::main::values::input_error::input_error; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_analysis::planner_strategy::main::capped_watermark_input::capped_watermark_input; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; use sqlbuild_python_project::native_host::models::ModelLookup; /// Reject capped producers feeding a microbatch model's watermark inputs. -#[inline] pub fn reject_capped_watermark_inputs<'py>( model: &Bound<'py, PyAny>, models_by_name: ModelLookup, functions_by_name: Option>, ) -> PyResult<()> { - lineage::reject_capped_watermark_inputs(model, models_by_name, functions_by_name) + let py: Python<'py> = model.py(); + if !microbatch_flag(model)? { + return Ok(()); + } + let functions_by_name: Bound<'py, PyAny> = match functions_by_name { + Some(functions) if functions.is_truthy()? => functions, + _ => PyDict::new(py).into_any(), + }; + let graph = ProjectLineage { + models_by_name, + functions_by_name, + }; + let name: String = model_name(model)?; + let inputs: Vec = watermark_names(model)?; + match capped_watermark_input(&graph, (&name, references(model)?), inputs) { + Ok(None) => Ok(()), + Ok(Some(capped)) => Err(input_error( + py, + ErrorId::CompileInput, + &capped.message(), + None, + )), + Err(failure) => Err(lineage_error(py, failure)), + } } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/resolve_lineage_reference.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/resolve_lineage_reference.rs index b1e42955b5..82d22edbd7 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/resolve_lineage_reference.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/resolve_lineage_reference.rs @@ -1,18 +1,26 @@ //! The reference one named input resolves to through model and SQL function references. -use crate::planner_strategy::_helpers::lineage; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_strategy::_helpers::lineage::ProjectLineage; +use crate::planner_strategy::_helpers::lineage::lineage_error; +use crate::planner_strategy::_helpers::lineage::references; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_analysis::planner_strategy::main::lineage_reference::lineage_reference; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; use sqlbuild_python_project::native_host::models::ModelLookup; /// The reference one named input resolves to through model and SQL function references. -#[inline] pub fn resolve_lineage_reference<'py>( model: &Bound<'py, PyAny>, input_name: &str, models_by_name: ModelLookup, functions_by_name: Bound<'py, PyAny>, ) -> PyResult> { - lineage::resolve_lineage_reference(model, input_name, models_by_name, functions_by_name) + let py: Python<'py> = model.py(); + let graph = ProjectLineage { + models_by_name, + functions_by_name, + }; + let name: String = model_name(model)?; + lineage_reference(&graph, (&name, references(model)?), input_name) + .map_err(|failure| lineage_error(py, failure)) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs index 56eeaa8c66..8a05cc4946 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs @@ -1,16 +1,48 @@ //! Apply CLI, project, and model batch limits in their documented precedence order. -use crate::planner_strategy::_helpers::microbatch_limits; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_strategy::_helpers::microbatch_limits::default_limit; +use crate::planner_strategy::_helpers::microbatch_limits::limited; +use crate::planner_strategy::_helpers::microbatch_limits::with_warnings; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::project::replace::replace; /// Apply CLI, project, and model batch limits in their documented precedence order. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, entry, inputs))] pub fn apply_configured_microbatch_limits<'py>( entry: &Bound<'py, PyAny>, inputs: &Bound<'py, PyAny>, ) -> PyResult> { - microbatch_limits::apply_configured_microbatch_limits(entry, inputs) + let py: Python<'py> = entry.py(); + let configured: Bound<'py, PyAny> = native_field(inputs, "max_microbatches")?; + let configured_action: Bound<'py, PyAny> = native_field(inputs, "microbatch_limit_action")?; + if native_field(inputs, "max_microbatches_is_override")?.is_truthy()? { + let error: Bound<'py, PyAny> = enum_member(py, RecordId::MicrobatchLimitAction, "ERROR")?; + return with_warnings(limited(entry, configured, error)?, Vec::new()); + } + let declared: Bound<'py, PyAny> = native_field(entry, "declared_microbatch_limit_action")?; + if declared.is_none() { + let maximum: Bound<'py, PyAny> = default_limit(entry, configured)?; + return with_warnings(limited(entry, maximum, configured_action)?, Vec::new()); + } + let mut warnings: Vec> = Vec::new(); + let mut entry: Bound<'py, PyAny> = entry.clone(); + if !configured.is_none() { + let (_, safety_warning) = limited(&entry, configured.clone(), configured_action.clone())?; + entry = replace( + &entry, + &[ + ("microbatch_safety_limit", configured), + ("microbatch_safety_limit_action", configured_action), + ], + )?; + warnings.extend(safety_warning); + } + let model_limit: Bound<'py, PyAny> = native_field(&entry, "microbatch_limit")?; + with_warnings(limited(&entry, model_limit, declared)?, warnings) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits_function.rs index 31f11af667..10e8cd5641 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits_function.rs @@ -1,11 +1,10 @@ //! `apply_configured_microbatch_limits` as a function of `module`. -use crate::planner_strategy::_helpers::microbatch_limits; +use crate::planner_strategy::main::microbatch_limits::apply_configured_microbatch_limits::apply_configured_microbatch_limits; /// `apply_configured_microbatch_limits` as a function of `module`. -#[inline] pub fn apply_configured_microbatch_limits_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - microbatch_limits::apply_configured_microbatch_limits_function(module) + pyo3::wrap_pyfunction!(apply_configured_microbatch_limits, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_microbatch_limit_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_microbatch_limit_function.rs index a5d0e2ad19..997c5de3cf 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_microbatch_limit_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_microbatch_limit_function.rs @@ -1,11 +1,10 @@ //! `apply_microbatch_limit` as a function of `module`. -use crate::planner_strategy::_helpers::microbatch_limits; +use crate::planner_strategy::_helpers::microbatch_limits::apply_microbatch_limit; /// `apply_microbatch_limit` as a function of `module`. -#[inline] pub fn apply_microbatch_limit_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - microbatch_limits::apply_microbatch_limit_function(module) + pyo3::wrap_pyfunction!(apply_microbatch_limit, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/register_microbatches.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/register_microbatches.rs index 046d394df8..c202e89d21 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/register_microbatches.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/register_microbatches.rs @@ -1,12 +1,11 @@ //! Register the microbatches functions and classes. -use crate::planner_strategy::_helpers::microbatches; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_strategy::_helpers::microbatches::microbatch_limit_warning; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the microbatches functions and classes. -#[inline] pub fn register_microbatches(module: &Bound<'_, PyModule>) -> PyResult<()> { - microbatches::register(module) + module.add_function(wrap_pyfunction!(microbatch_limit_warning, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/assemble_model_plan_entry.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/assemble_model_plan_entry.rs index a8e1cb9758..4a49ef367c 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/assemble_model_plan_entry.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/assemble_model_plan_entry.rs @@ -1,9 +1,7 @@ //! Build a model plan entry and its warnings from resolved plan decisions. use crate::planner_strategy::_helpers::model_entries; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyTuple; use sqlbuild_python_project::planner_strategy::models::ModelEntryDecisions; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/extract_seed_columns.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/extract_seed_columns.rs index 65e9d663b9..9c765aaa4b 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/extract_seed_columns.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/extract_seed_columns.rs @@ -1,13 +1,10 @@ //! Declared seed columns with types, as `ColumnInfo`. -use crate::planner_strategy::_helpers::model_entries; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_strategy::_helpers::model_entries::schema_columns; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyTuple; /// Declared seed columns with types, as `ColumnInfo`. -#[inline] pub fn extract_seed_columns<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - model_entries::extract_seed_columns(seed) + schema_columns(seed, true) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback.rs index e3d9a3fb48..2992ac7c77 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback.rs @@ -1,15 +1,15 @@ //! Resolve explicit or default lookback for one microbatch model. -use crate::planner_strategy::_helpers::model_entries; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_strategy::_helpers::model_entries::lookback_of; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// Resolve explicit or default lookback for one microbatch model. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, model, effective_grain = None))] pub fn resolve_microbatch_lookback( model: &Bound<'_, PyAny>, effective_grain: Option, ) -> PyResult> { - model_entries::resolve_microbatch_lookback(model, effective_grain) + lookback_of(model, effective_grain) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback_function.rs index b3ec92fffb..a55342d16d 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback_function.rs @@ -1,11 +1,10 @@ //! `resolve_microbatch_lookback` as a function of `module`. -use crate::planner_strategy::_helpers::model_entries; +use crate::planner_strategy::main::model_entries::resolve_microbatch_lookback::resolve_microbatch_lookback; /// `resolve_microbatch_lookback` as a function of `module`. -#[inline] pub fn resolve_microbatch_lookback_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - model_entries::resolve_microbatch_lookback_function(module) + pyo3::wrap_pyfunction!(resolve_microbatch_lookback, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/materialization_of.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/materialization_of.rs index 0d82ec45b1..45a13d553e 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/materialization_of.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/materialization_of.rs @@ -1,12 +1,18 @@ //! The `MaterializationType` of a model's `materialized` config: table when unset, custom when unknown. -use crate::planner_strategy::_helpers::strategy; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_changes::main::planner_enum::planner_enum; +use crate::planner_strategy::_helpers::values::{materialization_value, model_materialization}; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::RecordId; /// The `MaterializationType` of a model's `materialized` config: table when unset, custom when unknown. -#[inline] +#[pyfunction(name = "get_materialization_type")] +#[pyo3(signature = (model))] pub fn materialization_of<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - strategy::materialization_of(model) + planner_enum( + model.py(), + RecordId::MaterializationType, + materialization_value(model_materialization(model)?), + ) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs index a19d038880..57596372a8 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs @@ -1,12 +1,16 @@ //! Register the strategy functions and classes. -use crate::planner_strategy::_helpers::strategy; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::planner_strategy::_helpers::strategy::build_model_warnings; +use crate::planner_strategy::_helpers::strategy::resolve_schema_actions; +use crate::planner_strategy::main::strategy::materialization_of::materialization_of; +use crate::planner_strategy::main::strategy::resolve_model_plan_action::resolve_model_plan_action; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the strategy functions and classes. -#[inline] pub fn register_strategy(module: &Bound<'_, PyModule>) -> PyResult<()> { - strategy::register(module) + module.add_function(wrap_pyfunction!(materialization_of, module)?)?; + module.add_function(wrap_pyfunction!(resolve_model_plan_action, module)?)?; + module.add_function(wrap_pyfunction!(resolve_schema_actions, module)?)?; + module.add_function(wrap_pyfunction!(build_model_warnings, module)?) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/resolve_model_plan_action.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/resolve_model_plan_action.rs index 1a05091e9d..9d875a1b3b 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/resolve_model_plan_action.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/resolve_model_plan_action.rs @@ -1,17 +1,55 @@ //! The plan action and reason for one model. -use crate::planner_strategy::_helpers::strategy; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyTuple; +use crate::planner_changes::main::planner_enum::planner_enum; +use crate::planner_strategy::_helpers::strategy::action_value; +use crate::planner_strategy::_helpers::strategy::reason_value; +use crate::planner_strategy::_helpers::values::{ + change_facts, config_value, model_materialization, optional_string, +}; +use crate::planner_strategy::main::values::input_error::input_error; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyBool, PyTuple}; +use sqlbuild_analysis::planner_strategy::main::model_plan_action::model_plan_action; +use sqlbuild_analysis::planner_strategy::models::ModelStrategy; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_base::records::types::RecordId; /// The plan action and reason for one model. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, model, change_result, full_refresh))] pub fn resolve_model_plan_action<'py>( model: &Bound<'py, PyAny>, change_result: &Bound<'py, PyAny>, full_refresh: bool, ) -> PyResult> { - strategy::resolve_model_plan_action(model, change_result, full_refresh) + let py: Python<'py> = model.py(); + let enabled: Bound<'py, PyAny> = config_value(model, "enabled")?; + let strategy = ModelStrategy { + name: + sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name( + model, + )?, + disabled: enabled.is_instance_of::() && !enabled.is_truthy()?, + materialization: model_materialization(model)?, + incremental_strategy: optional_string(&config_value(model, "incremental_strategy")?)?, + }; + let (action, reason) = + model_plan_action(&strategy, &change_facts(change_result)?, full_refresh).map_err( + |error| { + input_error( + py, + ErrorId::PlannerInput, + &error.message(), + Some(error.code()), + ) + }, + )?; + PyTuple::new( + py, + [ + planner_enum(py, RecordId::PlanAction, action_value(action))?, + planner_enum(py, RecordId::PlanReason, reason_value(reason))?, + ], + ) } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs index 4bca20251b..22fc32fc49 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs @@ -1,12 +1,21 @@ //! The native error `class(message, code=code)`. -use crate::planner_strategy::_helpers::values; -use pyo3::prelude::PyErr; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; /// The native error `class(message, code=code)`. -#[inline] pub fn input_error(py: Python<'_>, class: ErrorId, message: &str, code: Option<&str>) -> PyErr { - values::input_error(py, class, message, code) + let raised: PyResult> = error_class(py, class).and_then(|class| { + let options: Bound<'_, PyDict> = PyDict::new(py); + if let Some(code) = code { + options.set_item("code", code)?; + } + class.call((message,), Some(&options)) + }); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/values/text.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/text.rs index 9e2cd8e8ca..6426d587e9 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/values/text.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/text.rs @@ -1,12 +1,8 @@ //! `str(value)` of a Python string or string enum. -use crate::planner_strategy::_helpers::values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// `str(value)` of a Python string or string enum. -#[inline] pub fn text(value: &Bound<'_, PyAny>) -> PyResult { - values::text(value) + value.extract() } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs index 912bb907c5..bf0e2ea08b 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs @@ -7,16 +7,12 @@ use crate::planner_resolution::main::safety::maximum_allowed_start::maximum_allo use crate::planner_strategy::main::lineage::resolve_lineage_reference::resolve_lineage_reference; use crate::planner_warehouse::_helpers::inspection::snapshot::inspection_concurrency; use crate::planner_warehouse::_helpers::inspection::watermarks::validate_watermark_column; -use crate::planner_warehouse::constants::CURSOR_ALGEBRA_ERROR; -use crate::planner_warehouse::constants::CURSOR_TYPE_INTEGER; -use crate::planner_warehouse::constants::CURSOR_TYPE_TIMESTAMP; -use crate::planner_warehouse::constants::DATE_VALUE; -use crate::planner_warehouse::constants::LOG_DEBUG_EVENT; -use crate::planner_warehouse::constants::PLANNER_LOGGER; -use crate::planner_warehouse::constants::QUERY_RECORD; -use crate::planner_warehouse::constants::RECORD_QUERY; -use crate::planner_warehouse::constants::RUN_BOUNDED; -use crate::planner_warehouse::constants::RUN_RECORDED_QUERY; +use crate::planner_warehouse::constants::{ + CURSOR_ALGEBRA_ERROR, CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, DATE_VALUE, LOG_DEBUG_EVENT, + PLANNER_LOGGER, QUERY_RECORD, RECORD_QUERY, RUN_BOUNDED, RUN_RECORDED_QUERY, +}; +use crate::planner_warehouse::main::inspection::adapter_cursor_sql::adapter_cursor_sql; +use crate::planner_warehouse::main::inspection::bound_task::bound_task; use crate::planner_warehouse::models::SnapshotRequest; use pyo3::IntoPyObject; use pyo3::exceptions::PyException; @@ -26,7 +22,6 @@ use sqlbuild_analysis::planner_changes::constants::FORWARD_ONLY_POLICY; use sqlbuild_analysis::planner_warehouse::main::cursor_bounds::cursor_bounds; use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_query_groups; use sqlbuild_analysis::planner_warehouse::main::cursor_query_identity::cursor_query_identity; -use sqlbuild_analysis::planner_warehouse::main::cursor_query_sql::cursor_query_sql; use sqlbuild_analysis::planner_warehouse::models::{CursorTagRequest, PhysicalCursorQuery}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -60,14 +55,13 @@ use sqlbuild_python_project::planner_phases::main::text::text; use sqlbuild_python_project::planner_phases::main::tuple::tuple; use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; -use std::cell::RefCell; use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; /// The adapter method that renders one cursor bound expression as text. -const CURSOR_TEXT_METHOD: &str = "render_cursor_text"; +pub(crate) const CURSOR_TEXT_METHOD: &str = "render_cursor_text"; /// Selection and policy inputs for cursor warehouse inspection. pub(crate) struct CursorInputs<'py> { @@ -140,7 +134,7 @@ fn parse<'py>( sqlbuild_python_front::cursor::main::scalars::parse::parse(raw, &text(py, cursor_type)) } -fn render<'py>(value: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn render<'py>(value: &Bound<'py, PyAny>) -> PyResult> { object(value.py(), render_value(value)?) } @@ -584,7 +578,7 @@ fn physical_queries(models: &[CursorModel<'_>]) -> Vec { } /// Run one physical cursor statement; a failure leaves only that relation unavailable. -fn read_cursor_query<'py>( +pub(crate) fn read_cursor_query<'py>( py: Python<'py>, sql: &str, (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -672,27 +666,6 @@ fn report_cursor_query( ) } -pub(crate) fn bound_task<'py>( - py: Python<'py>, - sql: String, - (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - let connection: Py = connection.clone().unbind(); - let execute: Py = execute.clone().unbind(); - Ok(PyCFunction::new_closure( - py, - None, - None, - move |args: &Bound<'_, PyTuple>, - _kwargs: Option<&Bound<'_, PyDict>>| - -> PyResult> { - let py = args.py(); - Ok(read_cursor_query(py, &sql, (connection.bind(py), execute.bind(py)))?.unbind()) - }, - )? - .into_any()) -} - fn completion_reporter<'py>( py: Python<'py>, queries: Vec, @@ -748,34 +721,6 @@ fn fan_out<'py>(query: &PhysicalCursorQuery, row: &Bound<'py, PyAny>) -> PyResul Ok(values) } -/// The query's MIN/MAX statement, each bound read as text the way the adapter renders it. -pub(crate) fn adapter_cursor_sql( - adapter: &Bound<'_, PyAny>, - query: &PhysicalCursorQuery, -) -> PyResult { - let py = adapter.py(); - let render = native_field(adapter, CURSOR_TEXT_METHOD)?; - let failure: RefCell> = RefCell::new(None); - let sql: String = cursor_query_sql(query, &|expression: &str| { - call_with( - &render, - &[ - ("expression", text(py, expression)), - ("cursor_type", text(py, &query.cursor_type)), - ], - ) - .and_then(|rendered| rendered.extract::()) - .unwrap_or_else(|error| { - failure.replace(Some(error)); - String::new() - }) - }); - match failure.into_inner() { - Some(error) => Err(error), - None => Ok(sql), - } -} - fn execute_queries<'py>( request: &SnapshotRequest<'py>, queries: &[PhysicalCursorQuery], diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs index f2f8383dbb..58eb73eae7 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs @@ -103,10 +103,3 @@ pub(crate) fn package_version(py: Python<'_>, package: &str) -> PyResult Err(error) => Err(error), } } - -/// `migration_fingerprint_work_info` as a function of `module`. -pub(crate) fn migration_fingerprint_work_info_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(migration_fingerprint_work_info, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs index 74b6241ade..5919658d1a 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs @@ -1,14 +1,11 @@ //! Gather a frozen point-in-time warehouse snapshot for planning. use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; -use crate::planner_warehouse::_helpers::inspection::cursors::CursorInputs; -use crate::planner_warehouse::_helpers::inspection::cursors::cursor_snapshots; -use crate::planner_warehouse::constants::ACTIVE_CATALOG; -use crate::planner_warehouse::constants::FUNCTION_NODE_TYPES; -use crate::planner_warehouse::constants::METADATA_NAME_FILTER_LIMIT; -use crate::planner_warehouse::constants::MIGRATE_FROM_CONFIG_KEY; -use crate::planner_warehouse::constants::RELATIONS_MAIN_COLUMNS; -use crate::planner_warehouse::constants::RELATIONS_MAIN_LIST; +use crate::planner_warehouse::_helpers::inspection::cursors::{CursorInputs, cursor_snapshots}; +use crate::planner_warehouse::constants::{ + ACTIVE_CATALOG, FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, MIGRATE_FROM_CONFIG_KEY, + RELATIONS_MAIN_COLUMNS, RELATIONS_MAIN_LIST, +}; use crate::planner_warehouse::models::SnapshotRequest; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -889,17 +886,3 @@ pub(crate) fn planner_metadata_name_filter<'py>( let selected = selected_keys.unwrap_or_else(|| py.None().into_bound(py)); optional_names(py, metadata_name_filter(project, &selected)?.as_ref()) } - -/// `gather_warehouse_snapshot` as a function of `module`. -pub(crate) fn gather_warehouse_snapshot_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(gather_warehouse_snapshot, module) -} - -/// `planner_metadata_name_filter` as a function of `module`. -pub(crate) fn planner_metadata_name_filter_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(planner_metadata_name_filter, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs index 85d7cad00b..c22cfb4a51 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs @@ -1,9 +1,9 @@ //! Validate reliable watermark column contracts before warehouse inspection. use crate::planner_strategy::main::cursor_types::classify_cursor_sql_type::classify_cursor_sql_type; -use crate::planner_warehouse::constants::CURSOR_TYPE_INTEGER; -use crate::planner_warehouse::constants::CURSOR_TYPE_TIMESTAMP; -use crate::planner_warehouse::constants::WATERMARK_CONTRACT_CODE; +use crate::planner_warehouse::constants::{ + CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, WATERMARK_CONTRACT_CODE, +}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods}; @@ -163,10 +163,3 @@ pub(crate) fn validate_watermark_column<'py>( (Some(WATERMARK_CONTRACT_CODE), None), )) } - -/// `watermark_type_is_compatible` as a function of `module`. -pub(crate) fn watermark_type_is_compatible_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(watermark_type_is_compatible, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs index 34841887b5..eabd9cf952 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs @@ -1,7 +1,6 @@ //! Complete missing source metadata in bounded, database-grouped batches. -use crate::planner_warehouse::_helpers::inspection::snapshot::columns_for; -use crate::planner_warehouse::_helpers::inspection::snapshot::list_relations; +use crate::planner_warehouse::_helpers::inspection::snapshot::{columns_for, list_relations}; use crate::planner_warehouse::constants::ACTIVE_CATALOG; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -295,17 +294,3 @@ pub(crate) fn semantic_source_columns<'py>( } Ok(result) } - -/// `list_source_candidates` as a function of `module`. -pub(crate) fn list_source_candidates_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(list_source_candidates, module) -} - -/// `semantic_source_columns` as a function of `module`. -pub(crate) fn semantic_source_columns_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(semantic_source_columns, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs index 5c82c1fb22..accc47be4b 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs @@ -1,14 +1,12 @@ //! Warehouse source relation listing and column gathering for planning. use crate::observations::main::capture_source_columns::capture_source_columns; -use crate::planner_warehouse::_helpers::inspection::snapshot::columns_for; -use crate::planner_warehouse::_helpers::inspection::snapshot::list_relations; -use crate::planner_warehouse::constants::COLUMN_INFO; -use crate::planner_warehouse::constants::METADATA_NAME_FILTER_LIMIT; -use crate::planner_warehouse::constants::RELATION_LOOKUP; +use crate::planner_warehouse::_helpers::inspection::snapshot::{columns_for, list_relations}; +use crate::planner_warehouse::constants::{ + COLUMN_INFO, METADATA_NAME_FILTER_LIMIT, RELATION_LOOKUP, +}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::pyfunction; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -228,31 +226,3 @@ pub(crate) fn source_inspection<'py>( } Ok((result, frozenset(py, &listed)?)) } - -pub(crate) fn gather_source_inspection<'py>( - project: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, - source_entries: Option>, -) -> PyResult> { - let (columns, listed) = source_inspection(project, (adapter, connection), source_entries)?; - Ok(PyTuple::new(project.py(), [columns.into_any(), listed])?.into_any()) -} - -#[pyfunction] -#[pyo3(signature = (*, project, adapter, connection, source_entries=None))] -pub(crate) fn gather_source_columns<'py>( - project: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, - source_entries: Option>, -) -> PyResult> { - Ok(source_inspection(project, (adapter, connection), source_entries)?.0) -} - -/// `gather_source_columns` as a function of `module`. -pub(crate) fn gather_source_columns_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(gather_source_columns, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs index f3e955a49f..30f0337351 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs @@ -2,8 +2,7 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::planner_warehouse::constants::MODEL_HOOK_CONFIG_KEYS; -use crate::planner_warehouse::constants::SQL_HOOK_ENTRY; +use crate::planner_warehouse::constants::{MODEL_HOOK_CONFIG_KEYS, SQL_HOOK_ENTRY}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; @@ -325,30 +324,3 @@ pub(crate) fn build_source_read_map<'py>( } Ok(result) } - -/// Add declared project sources that no selected SQL resource reads to a source map. -pub(crate) fn with_declared_source_reads<'py>( - project: &Bound<'py, PyAny>, - source_map: &Bound<'py, PyDict>, - source_names: &Bound<'py, PyAny>, -) -> PyResult> { - let mut missing: BTreeSet<_> = BTreeSet::new(); - for name in source_names.try_iter()? { - let name: String = name?.extract()?; - if !source_map.contains(&name)? { - missing.insert(name); - } - } - if missing.is_empty() { - return Ok(source_map.clone()); - } - let result = source_map.copy()?; - for source in sources(project)?.try_iter()? { - let entry = source_entry(&source?)?; - let name: String = native_field(&entry, "name")?.extract()?; - if missing.contains(&name) { - result.set_item(name, entry)?; - } - } - Ok(result) -} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs index 1f8797f702..5a3b9c02d1 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs @@ -1,76 +1,27 @@ //! Direct planner source freshness observation and the warnings for unknown freshness. -use crate::freshness::constants::PREVIEW_MESSAGE; use crate::freshness::main::build_direct_source_freshness_planning_result_function::build_direct_source_freshness_planning_result_function; use crate::freshness::main::build_direct_source_freshness_propagation_result_function::build_direct_source_freshness_propagation_result_function; -use crate::planner_warehouse::_helpers::inspection::snapshot::destination_nodes; -use crate::planner_warehouse::_helpers::inspection::snapshot::lower; +use crate::planner_warehouse::_helpers::inspection::snapshot::{destination_nodes, lower}; use crate::planner_warehouse::constants::RUN_SCOPE; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use sqlbuild_analysis::planner_scope::constants::MODEL; -use sqlbuild_analysis::planner_warehouse::main::freshness_unknown_message::freshness_unknown_message; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; -use sqlbuild_python_project::planner_phases::main::native::native; use sqlbuild_python_project::planner_phases::main::replace::replace; use sqlbuild_python_project::planner_phases::main::text::text; use sqlbuild_python_project::planner_phases::main::tuple::tuple; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::BTreeSet; -/// One warning per reason that left configured source freshness unknown. -pub(crate) fn source_freshness_unknown_warnings<'py>( - source_freshness: &Bound<'py, PyAny>, -) -> PyResult> { - let py = source_freshness.py(); - let mut reasons: Vec<(String, Vec)> = Vec::new(); - for unknown in native_field(source_freshness, "unknown_sources")? - .call_method0("values")? - .try_iter()? - { - let unknown = unknown?; - let reason: String = native_field(&unknown, "reason")?.str()?.extract()?; - if freshness_unknown_message(&reason, &[]).is_none() - || native_field(&unknown, "message")?.eq(PREVIEW_MESSAGE)? - { - continue; - } - let name: String = native_field(&unknown, "source_name")?.extract()?; - match reasons.iter_mut().find(|(existing, _)| *existing == reason) { - Some((_, names)) => names.push(name), - None => reasons.push((reason, vec![name])), - } - } - let mut warnings: Vec<_> = Vec::new(); - for (reason, names) in reasons { - if let Some(message) = freshness_unknown_message(&reason, &names) { - warnings.push(native( - py, - RecordId::PlanWarning, - &[ - ("model_name", py.None().into_bound(py)), - ( - "severity", - enum_member(py, RecordId::WarningSeverity, "WARNING")?, - ), - ("message", text(py, &message)), - ], - )?); - } - } - tuple(py, warnings) -} - fn state_database<'py>(project: &Bound<'py, PyAny>) -> PyResult> { for node in destination_nodes(project)? { let database = native_field(&sqlbuild_python_project::resource_views::main::resource_reads::destination::destination(&node)?, "database")?; @@ -207,10 +158,3 @@ pub(crate) fn build_planner_source_freshness_result<'py>( )?; replace(&result, &[("propagation", propagation)]) } - -/// `build_planner_source_freshness_result_of` as a function of `module`. -pub(crate) fn build_planner_source_freshness_result_of_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_planner_source_freshness_result_of, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs index ba807775a1..c808eadbc4 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs @@ -1,8 +1,8 @@ //! Selected-scope validation that external source tables exist in the warehouse. -use crate::planner_warehouse::constants::MISSING_SOURCE_TABLE_CODE; -use crate::planner_warehouse::constants::RUN_BOUNDED; -use crate::planner_warehouse::constants::WILDCARD_TABLE_CHARACTER; +use crate::planner_warehouse::constants::{ + MISSING_SOURCE_TABLE_CODE, RUN_BOUNDED, WILDCARD_TABLE_CHARACTER, +}; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; @@ -238,10 +238,3 @@ pub(crate) fn check_selected_source_tables_exist_binding<'py>( check_selected_source_tables_exist(project, (adapter, connection), scope, relations) } - -/// `check_selected_source_tables_exist_binding` as a function of `module`. -pub(crate) fn check_selected_source_tables_exist_binding_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(check_selected_source_tables_exist_binding, module) -} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs index b251f1b2c4..e60f0d0305 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs @@ -1,7 +1,6 @@ //! Public constants of planner warehouse. -use sqlbuild_host::entries::models::AdapterHost; -use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use sqlbuild_python_base::records::types::RecordId; pub(crate) const RELATIONS_MAIN_LIST: AdapterHost = AdapterHost::ListRelationsForInspection; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/adapter_cursor_sql.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/adapter_cursor_sql.rs index 09feaaccf6..e079335ca7 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/adapter_cursor_sql.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/adapter_cursor_sql.rs @@ -1,16 +1,38 @@ //! The query's MIN/MAX statement, each bound read as text the way the adapter renders it. -use crate::planner_warehouse::_helpers::inspection::cursors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_warehouse::_helpers::inspection::cursors::CURSOR_TEXT_METHOD; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; +use sqlbuild_analysis::planner_warehouse::main::cursor_query_sql::cursor_query_sql; use sqlbuild_analysis::planner_warehouse::models::PhysicalCursorQuery; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_phases::main::text::text; +use std::cell::RefCell; /// The query's MIN/MAX statement, each bound read as text the way the adapter renders it. -#[inline] pub fn adapter_cursor_sql( adapter: &Bound<'_, PyAny>, query: &PhysicalCursorQuery, ) -> PyResult { - cursors::adapter_cursor_sql(adapter, query) + let py = adapter.py(); + let render = native_field(adapter, CURSOR_TEXT_METHOD)?; + let failure: RefCell> = RefCell::new(None); + let sql: String = cursor_query_sql(query, &|expression: &str| { + call_with( + &render, + &[ + ("expression", text(py, expression)), + ("cursor_type", text(py, &query.cursor_type)), + ], + ) + .and_then(|rendered| rendered.extract::()) + .unwrap_or_else(|error| { + failure.replace(Some(error)); + String::new() + }) + }); + match failure.into_inner() { + Some(error) => Err(error), + None => Ok(sql), + } } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/bound_task.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/bound_task.rs index ada17151a7..30c54090f5 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/bound_task.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/bound_task.rs @@ -1,17 +1,27 @@ //! Bound task. -use crate::planner_warehouse::_helpers::inspection::cursors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::planner_warehouse::_helpers::inspection::cursors::read_cursor_query; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::types::{PyCFunction, PyDict, PyTuple}; /// Bound task. -#[inline] pub fn bound_task<'py>( py: Python<'py>, sql: String, (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { - cursors::bound_task(py, sql, (connection, execute)) + let connection: Py = connection.clone().unbind(); + let execute: Py = execute.clone().unbind(); + Ok(PyCFunction::new_closure( + py, + None, + None, + move |args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>| + -> PyResult> { + let py = args.py(); + Ok(read_cursor_query(py, &sql, (connection.bind(py), execute.bind(py)))?.unbind()) + }, + )? + .into_any()) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_redirected_cursor_snapshots.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_redirected_cursor_snapshots.rs index 1108675712..62154cdfdf 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_redirected_cursor_snapshots.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_redirected_cursor_snapshots.rs @@ -1,9 +1,7 @@ //! Cursor snapshots of models whose history lives in another relation or column. use crate::planner_warehouse::_helpers::inspection::snapshot; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict as KeywordDict; /// Cursor snapshots of models whose history lives in another relation or column. diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_warehouse_snapshot_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_warehouse_snapshot_function.rs index 36ea9ef67c..7d960618c4 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_warehouse_snapshot_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_warehouse_snapshot_function.rs @@ -1,11 +1,10 @@ //! `gather_warehouse_snapshot` as a function of `module`. -use crate::planner_warehouse::_helpers::inspection::snapshot; +use crate::planner_warehouse::_helpers::inspection::snapshot::gather_warehouse_snapshot; /// `gather_warehouse_snapshot` as a function of `module`. -#[inline] pub fn gather_warehouse_snapshot_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - snapshot::gather_warehouse_snapshot_function(module) + pyo3::wrap_pyfunction!(gather_warehouse_snapshot, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/migration_fingerprint_work_info_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/migration_fingerprint_work_info_function.rs index 82c63e6e98..7fd8b07545 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/migration_fingerprint_work_info_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/migration_fingerprint_work_info_function.rs @@ -1,11 +1,10 @@ //! `migration_fingerprint_work_info` as a function of `module`. -use crate::planner_warehouse::_helpers::inspection::fingerprint_cache; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::migration_fingerprint_work_info; /// `migration_fingerprint_work_info` as a function of `module`. -#[inline] pub fn migration_fingerprint_work_info_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - fingerprint_cache::migration_fingerprint_work_info_function(module) + pyo3::wrap_pyfunction!(migration_fingerprint_work_info, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/planner_metadata_name_filter_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/planner_metadata_name_filter_function.rs index ec24876457..193c3dd1b4 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/planner_metadata_name_filter_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/planner_metadata_name_filter_function.rs @@ -1,11 +1,10 @@ //! `planner_metadata_name_filter` as a function of `module`. -use crate::planner_warehouse::_helpers::inspection::snapshot; +use crate::planner_warehouse::_helpers::inspection::snapshot::planner_metadata_name_filter; /// `planner_metadata_name_filter` as a function of `module`. -#[inline] pub fn planner_metadata_name_filter_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - snapshot::planner_metadata_name_filter_function(module) + pyo3::wrap_pyfunction!(planner_metadata_name_filter, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/warehouse_snapshot.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/warehouse_snapshot.rs index 9b09c4c861..dbd37a07b1 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/warehouse_snapshot.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/warehouse_snapshot.rs @@ -2,9 +2,7 @@ use crate::planner_warehouse::_helpers::inspection::snapshot; use crate::planner_warehouse::models::SnapshotRequest; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Gather relations, columns, fingerprints and cursor bounds of the planned target schemas. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/watermark_type_is_compatible_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/watermark_type_is_compatible_function.rs index 16e7ed3167..bf5fecd895 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/watermark_type_is_compatible_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/watermark_type_is_compatible_function.rs @@ -1,11 +1,10 @@ //! `watermark_type_is_compatible` as a function of `module`. -use crate::planner_warehouse::_helpers::inspection::watermarks; +use crate::planner_warehouse::_helpers::inspection::watermarks::watermark_type_is_compatible; /// `watermark_type_is_compatible` as a function of `module`. -#[inline] pub fn watermark_type_is_compatible_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - watermarks::watermark_type_is_compatible_function(module) + pyo3::wrap_pyfunction!(watermark_type_is_compatible, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result.rs index e791a07726..a5f1839f35 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result.rs @@ -1,9 +1,7 @@ //! The source freshness evaluation of one inspection scope against the warehouse. use crate::planner_warehouse::_helpers::sources::source_freshness; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// The source freshness evaluation of one inspection scope against the warehouse. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result_of_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result_of_function.rs index 96617b10f0..ef83913e24 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result_of_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result_of_function.rs @@ -1,11 +1,10 @@ //! `build_planner_source_freshness_result_of` as a function of `module`. -use crate::planner_warehouse::_helpers::sources::source_freshness; +use crate::planner_warehouse::_helpers::sources::source_freshness::build_planner_source_freshness_result_of; /// `build_planner_source_freshness_result_of` as a function of `module`. -#[inline] pub fn build_planner_source_freshness_result_of_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - source_freshness::build_planner_source_freshness_result_of_function(module) + pyo3::wrap_pyfunction!(build_planner_source_freshness_result_of, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_source_read_map.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_source_read_map.rs index 42f6381727..2ea35fdb8f 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_source_read_map.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_source_read_map.rs @@ -1,10 +1,8 @@ //! Resolve source reads for selected SQL, or for `read_source_names` when given. use crate::planner_warehouse::_helpers::sources::source_deferral; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyDict as KeywordDict; +use pyo3::prelude::{Bound, PyResult}; +use pyo3::types::{PyDict, PyDict as KeywordDict}; /// Resolve source reads for selected SQL, or for `read_source_names` when given. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist.rs index c25e455328..6ad31a35c8 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist.rs @@ -1,9 +1,7 @@ //! Raise S405 when a selected resource reads an external source table that does not exist. use crate::planner_warehouse::_helpers::sources::source_tables; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Raise S405 when a selected resource reads an external source table that does not exist. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist_binding_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist_binding_function.rs index d8248caa1f..7c84f33321 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist_binding_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist_binding_function.rs @@ -1,11 +1,10 @@ //! `check_selected_source_tables_exist_binding` as a function of `module`. -use crate::planner_warehouse::_helpers::sources::source_tables; +use crate::planner_warehouse::_helpers::sources::source_tables::check_selected_source_tables_exist_binding; /// `check_selected_source_tables_exist_binding` as a function of `module`. -#[inline] pub fn check_selected_source_tables_exist_binding_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - source_tables::check_selected_source_tables_exist_binding_function(module) + pyo3::wrap_pyfunction!(check_selected_source_tables_exist_binding, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns.rs index 3947717f6e..d3d36b5bb4 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns.rs @@ -1,18 +1,17 @@ //! Gather source columns. -use crate::planner_warehouse::_helpers::sources::source_columns; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_warehouse::_helpers::sources::source_columns::source_inspection; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyDict; -/// Gather source columns. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project, adapter, connection, source_entries=None))] pub fn gather_source_columns<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, source_entries: Option>, ) -> PyResult> { - source_columns::gather_source_columns(project, adapter, connection, source_entries) + Ok(source_inspection(project, (adapter, connection), source_entries)?.0) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns_function.rs index 76ae5e4d3a..1088a18001 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns_function.rs @@ -1,11 +1,10 @@ //! `gather_source_columns` as a function of `module`. -use crate::planner_warehouse::_helpers::sources::source_columns; +use crate::planner_warehouse::main::sources::gather_source_columns::gather_source_columns; /// `gather_source_columns` as a function of `module`. -#[inline] pub fn gather_source_columns_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - source_columns::gather_source_columns_function(module) + pyo3::wrap_pyfunction!(gather_source_columns, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_inspection.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_inspection.rs index 0153d0510a..62fe3bdf29 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_inspection.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_inspection.rs @@ -1,17 +1,16 @@ //! Gather source inspection. -use crate::planner_warehouse::_helpers::sources::source_columns; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::planner_warehouse::_helpers::sources::source_columns::source_inspection; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyTuple; /// Gather source inspection. -#[inline] pub fn gather_source_inspection<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, source_entries: Option>, ) -> PyResult> { - source_columns::gather_source_inspection(project, adapter, connection, source_entries) + let (columns, listed) = source_inspection(project, (adapter, connection), source_entries)?; + Ok(PyTuple::new(project.py(), [columns.into_any(), listed])?.into_any()) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/list_source_candidates_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/list_source_candidates_function.rs index bd40139eaa..b87ee0d23f 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/list_source_candidates_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/list_source_candidates_function.rs @@ -1,11 +1,10 @@ //! `list_source_candidates` as a function of `module`. -use crate::planner_warehouse::_helpers::sources::semantic_sources; +use crate::planner_warehouse::_helpers::sources::semantic_sources::list_source_candidates; /// `list_source_candidates` as a function of `module`. -#[inline] pub fn list_source_candidates_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - semantic_sources::list_source_candidates_function(module) + pyo3::wrap_pyfunction!(list_source_candidates, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns.rs index 71750cb10b..89a3594a63 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns.rs @@ -1,10 +1,8 @@ //! Reuse collected columns and inspect only selected, unresolved table sources. use crate::planner_warehouse::_helpers::sources::semantic_sources; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyDict as KeywordDict; +use pyo3::prelude::{Bound, PyResult}; +use pyo3::types::{PyDict, PyDict as KeywordDict}; /// Reuse collected columns and inspect only selected, unresolved table sources. #[inline] diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns_function.rs index 9579b5c5fb..8d85cb0511 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns_function.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns_function.rs @@ -1,11 +1,10 @@ //! `semantic_source_columns` as a function of `module`. -use crate::planner_warehouse::_helpers::sources::semantic_sources; +use crate::planner_warehouse::_helpers::sources::semantic_sources::semantic_source_columns; /// `semantic_source_columns` as a function of `module`. -#[inline] pub fn semantic_source_columns_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - semantic_sources::semantic_source_columns_function(module) + pyo3::wrap_pyfunction!(semantic_source_columns, module) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/source_freshness_unknown_warnings.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/source_freshness_unknown_warnings.rs index ffdb9e6ece..db67581c12 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/source_freshness_unknown_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/source_freshness_unknown_warnings.rs @@ -1,14 +1,54 @@ //! One warning per reason that left configured source freshness unknown. -use crate::planner_warehouse::_helpers::sources::source_freshness; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::freshness::constants::PREVIEW_MESSAGE; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::planner_warehouse::main::freshness_unknown_message::freshness_unknown_message; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; /// One warning per reason that left configured source freshness unknown. -#[inline] pub fn source_freshness_unknown_warnings<'py>( source_freshness: &Bound<'py, PyAny>, ) -> PyResult> { - source_freshness::source_freshness_unknown_warnings(source_freshness) + let py = source_freshness.py(); + let mut reasons: Vec<(String, Vec)> = Vec::new(); + for unknown in native_field(source_freshness, "unknown_sources")? + .call_method0("values")? + .try_iter()? + { + let unknown = unknown?; + let reason: String = native_field(&unknown, "reason")?.str()?.extract()?; + if freshness_unknown_message(&reason, &[]).is_none() + || native_field(&unknown, "message")?.eq(PREVIEW_MESSAGE)? + { + continue; + } + let name: String = native_field(&unknown, "source_name")?.extract()?; + match reasons.iter_mut().find(|(existing, _)| *existing == reason) { + Some((_, names)) => names.push(name), + None => reasons.push((reason, vec![name])), + } + } + let mut warnings: Vec<_> = Vec::new(); + for (reason, names) in reasons { + if let Some(message) = freshness_unknown_message(&reason, &names) { + warnings.push(native( + py, + RecordId::PlanWarning, + &[ + ("model_name", py.None().into_bound(py)), + ( + "severity", + enum_member(py, RecordId::WarningSeverity, "WARNING")?, + ), + ("message", text(py, &message)), + ], + )?); + } + } + tuple(py, warnings) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/with_declared_source_reads.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/with_declared_source_reads.rs index 6be27897b5..a7f8edbd4c 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/with_declared_source_reads.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/with_declared_source_reads.rs @@ -1,17 +1,35 @@ //! Add declared project sources that no selected SQL resource reads to a source map. -use crate::planner_warehouse::_helpers::sources::source_deferral; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::project::sources::sources; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; +use std::collections::BTreeSet; /// Add declared project sources that no selected SQL resource reads to a source map. -#[inline] pub fn with_declared_source_reads<'py>( project: &Bound<'py, PyAny>, source_map: &Bound<'py, PyDict>, source_names: &Bound<'py, PyAny>, ) -> PyResult> { - source_deferral::with_declared_source_reads(project, source_map, source_names) + let mut missing: BTreeSet<_> = BTreeSet::new(); + for name in source_names.try_iter()? { + let name: String = name?.extract()?; + if !source_map.contains(&name)? { + missing.insert(name); + } + } + if missing.is_empty() { + return Ok(source_map.clone()); + } + let result = source_map.copy()?; + for source in sources(project)?.try_iter()? { + let entry = source_entry(&source?)?; + let name: String = native_field(&entry, "name")?.extract()?; + if missing.contains(&name) { + result.set_item(name, entry)?; + } + } + Ok(result) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index b013e1b08d..80669e53eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -57,8 +57,7 @@ use sqlbuild_python_planner::consumption::main::held_back_text::held_back_text; use sqlbuild_python_planner::consumption::main::policy_line::policy_line; use sqlbuild_python_planner::consumption::main::prepare::prepare as prepare_consumption; use sqlbuild_python_planner::consumption::main::run_reason_text::run_reason_text; -use sqlbuild_python_planner::consumption::models::ConsumptionRecorder; -use sqlbuild_python_planner::consumption::models::Prepared; +use sqlbuild_python_planner::consumption::models::{ConsumptionRecorder, Prepared}; use sqlbuild_python_planner::freshness::main::append_freshness_records::append_freshness_records; use sqlbuild_python_planner::observations::main::append_build_observations::append_build_observations; use sqlbuild_python_planner::planner_resolution::main::cursor::type_cursor_overrides::type_cursor_overrides; diff --git a/fensu.toml b/fensu.toml index f0d95e486e..2673f3ae58 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2276,22 +2276,27 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs=6", "crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs=3", "crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs=1", - "crates/sqlbuild-python-planner/src/freshness/_helpers/publication.rs=9", - "crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs=3", - "crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs=2", + "crates/sqlbuild-python-planner/src/freshness/_helpers/publication.rs=6", + "crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs=1", + "crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs=0", + "crates/sqlbuild-python-planner/src/freshness/main/append_freshness_records.rs=3", + "crates/sqlbuild-python-planner/src/freshness/main/presumed_utc_datetime.rs=2", + "crates/sqlbuild-python-planner/src/freshness/main/row_record.rs=2", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs=4", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/payloads.rs=4", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs=4", - "crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs=1", + "crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs=0", "crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs=1", "crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs=5", + "crates/sqlbuild-python-planner/src/microbatches/main/timestamp.rs=1", "crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs=5", - "crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs=10", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs=9", "crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs=1", "crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs=7", "crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs=5", "crates/sqlbuild-python-planner/src/plan_output/_helpers/python_identity.rs=2", "crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs=1", + "crates/sqlbuild-python-planner/src/plan_output/main/resolve_effective_microbatch_batch_size.rs=1", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs=5", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs=2", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs=4", @@ -2299,13 +2304,17 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs=1", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/values.rs=1", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs=5", - "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs=4", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs=3", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs=1", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs=2", - "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs=2", - "crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs=2", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs=0", + "crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_test_planning.rs=2", + "crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_command_scope.rs=1", + "crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs=0", "crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs=6", "crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs=1", + "crates/sqlbuild-python-planner/src/planner_identity/main/hook_version_hashes.rs=1", + "crates/sqlbuild-python-planner/src/planner_identity/main/merge_recomputed_model_changes.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs=3", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs=3", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs=1", @@ -2330,8 +2339,9 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs=7", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs=2", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs=6", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/projections.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/models.rs=1", "crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs=2", "crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs=1", @@ -2352,12 +2362,14 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs=1", "crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs=1", "crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs=1", - "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs=6", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs=4", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs=8", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs=2", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs=1", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs=3", - "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs=1", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs=0", + "crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs=2", + "crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs=1", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs=14", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs=6", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs=3", @@ -2365,9 +2377,10 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs=3", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs=1", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs=1", - "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs=6", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs=5", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs=5", "crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs=1", + "crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/source_freshness_unknown_warnings.rs=1", "crates/sqlbuild-python-planner/src/planner_warehouse/models.rs=8", "crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs=0", "crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs=1", From d7bd6aa7a15cd07ff85b8cff80bdc34328e119be Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 18:37:55 +0000 Subject: [PATCH 4/5] refactor: fit the planner crate to the lint, duplicate and fallback checks --- crates/README.md | 5 +- .../_helpers/column/column_candidates.rs | 2 +- .../_helpers/column/column_overlay.rs | 2 +- .../_helpers/column/columns.rs | 2 +- .../_helpers/discovery/mod.rs | 2 +- .../{discovery.rs => model_discovery.rs} | 0 .../_helpers/model/model_planning.rs | 2 +- .../_helpers/projections/mod.rs | 2 +- .../{projections.rs => query_shapes.rs} | 0 .../main/register_projections.rs | 4 +- fensu.toml | 108 +++++++- tests/native_field_fallbacks.txt | 233 +++++++++--------- 12 files changed, 235 insertions(+), 127 deletions(-) rename crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/{discovery.rs => model_discovery.rs} (100%) rename crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/{projections.rs => query_shapes.rs} (100%) diff --git a/crates/README.md b/crates/README.md index d6edeeaef9..5c7d6d822c 100644 --- a/crates/README.md +++ b/crates/README.md @@ -31,14 +31,15 @@ tested and benchmarked with `cargo test` alone. | `sqlbuild-python-base` | The base of the PyO3 bindings: native record, enum and error classes with their specs, registration, construction and field reads; keyword calls, imports, conversions and panics at the Python boundary; host entry calls; the process allocator; and the shared runtime the binding modules build on: the native store, compile context, diagnostics, progress and terminal text, warehouse access and orchestration reads. | | `sqlbuild-python-front` | The front of the compile pipeline as PyO3 bindings, on top of `sqlbuild-python-base`: project discovery, model configuration and headers, scopes, macro rendering and authored values, SQL text, model attachment and the compile driver, cursors, contract and refactor facts, and the warehouse state tables of consumption, observations and audit results. | | `sqlbuild-python-project` | The compiled project as PyO3 bindings, on top of `sqlbuild-python-front`: the project host and the Python views of compiled models, inputs and resources, the compiled-project snapshot, analysis classes and the analysis session, the project graph, references, lineage, fingerprints, contracts and semantic shapes, audit rendering, and the planner's identities, scope, selection, phases, fixture strategy and SQL test planning. | -| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. | +| `sqlbuild-python-planner` | The plan layer as PyO3 bindings, on top of `sqlbuild-python-project`: execution plans and their entries, planner phases, change detection, migrations, reference and cursor resolution, scenarios, strategy and warehouse inspection, plus freshness, consumption, microbatches, observations, lifecycle events and plan output. | +| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-planner`, `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. | ## Layering Dependencies point one way, from the top of this graph to the bottom: ```text -sqlbuild-python -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base +sqlbuild-python -> sqlbuild-python-planner -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base -> sqlbuild-project -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs index 244560de76..8e0c01fe50 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs @@ -1,7 +1,7 @@ //! Gather live candidate evidence without changing adapter inspection ownership. use crate::planner_migrations::_helpers::column::column_objects::{config, definition_hash, lower}; -use crate::planner_migrations::_helpers::projections::projections::parse_query_shape; +use crate::planner_migrations::_helpers::projections::query_shapes::parse_query_shape; use crate::planner_migrations::models::{ColumnCandidate, ColumnMigrationInputs, MigrationPhase}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs index 1b6155a88e..3d9eecefc4 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs @@ -3,7 +3,7 @@ use crate::planner_migrations::_helpers::column::column_objects::{ config, definition_hash, lower, native, replace, }; -use crate::planner_migrations::_helpers::projections::projections::renames_explain_change; +use crate::planner_migrations::_helpers::projections::query_shapes::renames_explain_change; use crate::planner_migrations::models::ColumnCandidate; use crate::planner_migrations::models::{ColumnMigrationInputs, MigrationPhase}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs index 87f78771c4..401798185f 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs @@ -70,7 +70,7 @@ pub(crate) fn identical_renames<'py>( for (old, new) in pairs { renames.set_item(old, new)?; } - crate::planner_migrations::_helpers::projections::projections::explain( + crate::planner_migrations::_helpers::projections::query_shapes::explain( &previous, ¤t, &renames.into_any(), diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/mod.rs index bdbabf9496..b4f9da8864 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/mod.rs @@ -1,5 +1,5 @@ -pub(crate) mod discovery; pub(crate) mod discovery_events; pub(crate) mod discovery_models; pub(crate) mod discovery_requests; +pub(crate) mod model_discovery; pub(crate) mod state_inspection; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs similarity index 100% rename from crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs index 31cb8c9989..4cc0a60a9a 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs @@ -2,7 +2,7 @@ use crate::planner_changes::main::plan_reference_renames::plan_reference_renames; use crate::planner_migrations::_helpers::column::column_objects::{config, replace}; -use crate::planner_migrations::_helpers::discovery::discovery::discover_model_migrations; +use crate::planner_migrations::_helpers::discovery::model_discovery::discover_model_migrations; use crate::planner_migrations::_helpers::planning::planning_origin::{ require_same_database, resolve_origin, }; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/mod.rs index 4db25c4cea..76756461fb 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/mod.rs @@ -1,3 +1,3 @@ pub(crate) mod fingerprint; pub(crate) mod getters; -pub(crate) mod projections; +pub(crate) mod query_shapes; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/projections.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/query_shapes.rs similarity index 100% rename from crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/projections.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/query_shapes.rs diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_projections.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_projections.rs index 5d3d1d9648..275627e07b 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_projections.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_projections.rs @@ -1,7 +1,7 @@ //! Register the projections functions and classes. -use crate::planner_migrations::_helpers::projections::projections::parse_query_shape; -use crate::planner_migrations::_helpers::projections::projections::renames_explain_change; +use crate::planner_migrations::_helpers::projections::query_shapes::parse_query_shape; +use crate::planner_migrations::_helpers::projections::query_shapes::renames_explain_change; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; diff --git a/fensu.toml b/fensu.toml index 2673f3ae58..88cb94cfe6 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1672,7 +1672,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 114941 } +thresholds = { max_crate_code_lines = 114940 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] @@ -2322,10 +2322,10 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs=1", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs=6", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_models.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs=2", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs=9", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs=5", @@ -2340,7 +2340,7 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs=2", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs=6", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs=1", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/projections.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/query_shapes.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/models.rs=1", "crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs=2", @@ -3343,7 +3343,7 @@ reason = "Each public entry forwards to its own helper with the same arguments." [[dupes.allowlist]] paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs", "crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs"] -reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." +reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs"] @@ -3629,6 +3629,106 @@ reason = "Each converts its own input into its own native or Python value; only paths = ["crates/sqlbuild-python-front/src/render/models.rs", "crates/sqlbuild-python-project/src/analysis_session/models.rs", "crates/sqlbuild-python-project/src/compiled_project/models.rs"] reason = "Each positional record converts its own fields; models.rs holds only records, so these are the conversions positional_record! writes, written out once per record." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/oracles.rs", "crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs", "crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs", "crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs", "crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs", "crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs"] +reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs", "crates/sqlbuild-python-planner/src/plan_output/main/dict.rs", "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs"] +reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/main/register_envelopes.rs", "crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs"] +reason = "Each bindings module registers its own functions and classes with the native module." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_identity/main/or_empty.rs", "crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs", "crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/getters.rs"] +reason = "Each matches its own native class's attribute names to its own fields; only the cast-and-dispatch shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/getters.rs"] +reason = "Each matches its own native class's attribute names to its own fields; only the cast-and-dispatch shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/microbatches/main/register_statements.rs", "crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_command_plans.rs", "crates/sqlbuild-python-planner/src/planner_identity/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_windows.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/getters.rs"] +reason = "Each matches its own native class's attribute names to its own fields; only the cast-and-dispatch shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/getters.rs"] +reason = "Each matches its own native class's attribute names to its own fields; only the cast-and-dispatch shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs"] +reason = "Each writes the entries of its own plan section with its own keys; only mapping over the entries is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/contexts.rs", "crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs"] +reason = "runtime_field and warehouse_field are class_field fixed to one planner class; moving their call sites to class_field is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs", "crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs"] +reason = "The full relations context reads the connection and known source columns from keywords and the static one fills them with empty values; only the keyword parsing is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/config_str.rs"] +reason = "Each planner area reads one model config value with its own copy of the reader; sharing one reader across the planner areas is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs"] +reason = "Each maps its own enum's variants to its own labels; only the match shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mock_reads.rs", "crates/sqlbuild-analysis/src/required_columns/_helpers/relations.rs", "crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs"] +reason = "Each replaces or appends a value in its own ordered entry list; the types are separate and live in separate crates." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs"] +reason = "Each Python entry reads its own keyword arguments and calls its own resolver; only the keyword parsing is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs"] +reason = "Each writes its own record's fields as a JSON dict; only building the dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs"] +reason = "One renders the insert statement for the events and the other the lookup of their IDs; only the events-table set-up is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_migrations/models.rs"] +reason = "Each looks up its own map by relation key with its own default; only the lookup is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs"] +reason = "Each reads and resets its own work counters; only the take-and-reset shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs"] +reason = "One observes one configured source and the other a batch of adapter sources; only building the observer is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs"] +reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." + [[dupes.allowlist]] paths = ["crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs", "crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs"] reason = "Each is one rule family's entry point: return early when no rule is selected, then run its own layout, module and import checks. Parsing, project modules and import resolution share files_naming, project_modules and rejected_import_faults." diff --git a/tests/native_field_fallbacks.txt b/tests/native_field_fallbacks.txt index 26f2188a9b..3be8268760 100644 --- a/tests/native_field_fallbacks.txt +++ b/tests/native_field_fallbacks.txt @@ -188,9 +188,9 @@ crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs Dec crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs ScopeKind.value crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs EventExporterCommandScope.cached_extensions crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs TransientStatusReporter.complete -crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs SourceFreshnessAgeStatus.value -crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs SourceFreshnessStrategy.value -crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs SourceFreshnessValueKind.value +crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs SourceFreshnessAgeStatus.value +crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs SourceFreshnessStrategy.value +crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs SourceFreshnessValueKind.value crates/sqlbuild-python-base/src/frontier/models.rs BackfillAction.name crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs BindingCatalog.analysis_shapes crates/sqlbuild-python-base/src/frontier/models.rs BindingCatalog.analysis_shapes @@ -261,117 +261,124 @@ crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs ConnectionProg crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs ConnectionProgressReporter.on_connection_error crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs ConnectionProgressReporter.on_connection_start crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs PlanningProgressReporter.on_progress -crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs MigrationStateInspection.inspect_migration_events -crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs PlannerRuntime.project -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs MaterializationType.value -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs PlannerRelationsContext.function_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs PlannerRelationsContext.model_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs PlannerRelationsContext.python_source_read_map -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs PlannerRelationsContext.seed_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs PlannerRelationsContext.source_map -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs PlannerRelationsContext.source_read_map -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.function_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.model_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.python_source_read_map -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.seed_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.source_map -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.source_read_map -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.source_warehouse_columns -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.star_exclude_keyword -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs ColumnMigrationDecision.blocks_build -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs ColumnMigrationDecision.renames -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs ColumnMigrationEvent.renamed -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs PlannerRuntime.adapter -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs PlannerRuntime.project -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs ColumnMigrationDecision.renames -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs MigrationStateInspection.inspect_column_events -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs MigrationStateInspection.model_fingerprint -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs PlannerRuntime.project -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs MigrationStateInspection.inspect_relations -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs MigrationStateInspection.orphan_fingerprints -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs MigrationStateInspection.inspect_schemas -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs MigrationStateInspection.named_fingerprint -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs PlannerRuntime.adapter -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs MigrationFingerprintCache.fingerprint -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs MigrationFingerprintCache.fingerprint -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs MigrationDecision.moves_data -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs MigrationStateInspection.model_fingerprint -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs PlannerRuntime.project -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs MigrationDecision.moves_data -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs MigrationStateInspection.inspect_relations -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs MigrationStateInspection.inspect_schemas -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs MigrationDecision.moves_data -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs OldNameViewHistory.status -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs PlannerRuntime.adapter -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs MigrationStateInspection.inspect_old_name_views -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs PlannerRuntime.project -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs PlannerRuntime.project -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs MigrationStateInspection.named_fingerprint -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs PlannerRuntime.adapter -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs PlannerRelationsContext.source_warehouse_columns -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs PlannerWarehouseState.inspection_relations -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs PlannerRuntime.adapter -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs SourceFreshnessUnknownReason.value -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs PlannerRuntime.adapter -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs PlannerRuntime.project -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs PlannerWarehouseState.snapshot -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs RetentionDecreasePolicy.value -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs TableTypeDowngradePolicy.value -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs TableTypeSource.value -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs TableType.value -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs TimeTravelRetentionSource.value -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs MigrationDecision.moves_data -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs PlannerRelationsContext.source_warehouse_columns -crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs FutureCursorAction.value -crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs CursorOverrides.typed_as -crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs DbtCompileReferenceResolver.resolve_reference -crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs ScenarioArtifactKind.value -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs PlannerRelationsContext.model_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs PlannerRelationsContext.seed_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs PlannerRelationsContext.source_read_map +crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs MigrationStateInspection.inspect_migration_events +crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs MaterializationType.value +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs PlannerRelationsContext.function_locations +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs PlannerRelationsContext.model_locations +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs PlannerRelationsContext.python_source_read_map +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs PlannerRelationsContext.seed_locations +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs PlannerRelationsContext.source_map +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs PlannerRelationsContext.source_read_map +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.function_locations +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.model_locations +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.python_source_read_map +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.seed_locations +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.source_map +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.source_read_map +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.source_warehouse_columns +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.star_exclude_keyword +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs ColumnMigrationDecision.blocks_build +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs ColumnMigrationDecision.renames +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs ColumnMigrationEvent.renamed +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs PlannerRuntime.adapter +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs ColumnMigrationDecision.renames +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs MigrationStateInspection.inspect_column_events +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs MigrationStateInspection.model_fingerprint +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs MigrationStateInspection.inspect_relations +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs MigrationStateInspection.orphan_fingerprints +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs MigrationStateInspection.inspect_schemas +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs MigrationStateInspection.named_fingerprint +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs PlannerRuntime.adapter +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs MigrationFingerprintCache.fingerprint +crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs MigrationFingerprintCache.fingerprint +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs MigrationDecision.moves_data +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs MigrationStateInspection.model_fingerprint +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs MigrationDecision.moves_data +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs MigrationStateInspection.inspect_relations +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs MigrationStateInspection.inspect_schemas +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs MigrationDecision.moves_data +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs OldNameViewHistory.status +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs PlannerRuntime.adapter +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs MigrationStateInspection.inspect_old_name_views +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs MigrationStateInspection.named_fingerprint +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs PlannerRuntime.adapter +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs PlannerRelationsContext.source_warehouse_columns +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs PlannerWarehouseState.inspection_relations +crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs PlannerRuntime.adapter +crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs SourceFreshnessUnknownReason.value +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs PlannerRuntime.adapter +crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effective_target_config.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs PlannerWarehouseState.snapshot +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs RetentionDecreasePolicy.value +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs TableTypeDowngradePolicy.value +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs TableTypeSource.value +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs TableType.value +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs TimeTravelRetentionSource.value +crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs MigrationDecision.moves_data +crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs PlannerRelationsContext.source_warehouse_columns +crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs FutureCursorAction.value +crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs CursorOverrides.typed_as +crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs DbtCompileReferenceResolver.resolve_reference +crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs ScenarioArtifactKind.value +crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs PlannerRelationsContext.model_locations +crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs PlannerRelationsContext.model_locations +crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs PlannerRelationsContext.seed_locations +crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs PlannerRelationsContext.seed_locations +crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs PlannerRelationsContext.source_read_map crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_values.rs CompiledResourceType.value -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs PlanEntryBuildInputs.max_microbatches -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs PlanEntryBuildInputs.max_microbatches_is_override -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs PlanEntryBuildInputs.microbatch_limit_action -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs TableType.value -crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs PlannerRelationsContext.source_read_map -crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs PlannerRelationsContext.listed_source_names -crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs PlannerRelationsContext.source_read_map -crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs GraphResourceKind.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs CliStyle.use_color -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs ColumnMigrationDecision.label -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs MigrationCompatibility.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs MigrationDecision.blocks_build -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs MigrationDecision.checks_compatibility -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs MigrationDecision.label -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs MigrationDiscovery.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs MigrationPromotion.label -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs RetentionDirection.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs RetentionPlanPhase.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs SourceWriteStrategy.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs BackfillAction.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs PlanAction.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs PlanReason.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs PythonIdentityStatus.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs PythonRunPhase.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs SchemaChangeKind.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs SourceWriteStrategy.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs WarningSeverity.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs BackfillAction.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs ColumnMigrationDecision.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs FutureCursorAction.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs MaterializationType.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs MicrobatchLimitAction.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs MigrationCompatibility.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs MigrationDecision.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs MigrationDiscovery.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs MigrationPromotion.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs OldNameViewAction.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs PlanAction.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs PlanReason.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs PythonIdentityStatus.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs PythonRunPhase.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs WarningSeverity.value +crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatches.rs PlanEntryBuildInputs.max_microbatches +crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs PlanEntryBuildInputs.max_microbatches +crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs PlanEntryBuildInputs.max_microbatches_is_override +crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs PlanEntryBuildInputs.microbatch_limit_action +crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs PlanEntryBuildInputs.microbatch_limit_action +crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs TableType.value +crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs PlannerRelationsContext.source_read_map +crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs PlannerRelationsContext.listed_source_names +crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs PlannerRelationsContext.source_read_map +crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs GraphResourceKind.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs CliStyle.use_color +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs ColumnMigrationDecision.label +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs MigrationCompatibility.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs MigrationDecision.blocks_build +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs MigrationDecision.checks_compatibility +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs MigrationDecision.label +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs MigrationDiscovery.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs MigrationPromotion.label +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs RetentionDirection.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs RetentionPlanPhase.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs SourceWriteStrategy.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs BackfillAction.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs PlanAction.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs PlanReason.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs PythonIdentityStatus.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs PythonRunPhase.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs SchemaChangeKind.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs SourceWriteStrategy.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs WarningSeverity.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs BackfillAction.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs ColumnMigrationDecision.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs FutureCursorAction.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs MaterializationType.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs MicrobatchLimitAction.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs MigrationCompatibility.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs MigrationDecision.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs MigrationDiscovery.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs MigrationPromotion.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs OldNameViewAction.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs PlanAction.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs PlanReason.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs PythonIdentityStatus.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs PythonRunPhase.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs WarningSeverity.value crates/sqlbuild-python-base/src/presentation_text/_helpers/functions.rs CliStyle.use_color crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs MacroArgumentError.detail crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs MacroArgumentError.file_path From 94b166142190e311be8f03393f2d838b67fe3367 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:04:40 +0000 Subject: [PATCH 5/5] fix: key the dynamic planner field reads to their call site and correct the register allowlist reason --- .../src/planner_scope/main/resolve_planner_scopes.rs | 1 - fensu.toml | 6 +----- tests/native_field_fallbacks.txt | 4 ++-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes.rs b/crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes.rs index 73e1b9f8b0..1c26f9b08c 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes.rs @@ -12,7 +12,6 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; #[pyfunction] #[pyo3(signature = (*, project, selection, policies))] -/// The selected, stale-warning and inspection scopes `selection` resolves over `project`. pub fn resolve_planner_scopes<'py>( project: &Bound<'py, PyAny>, selection: &Bound<'py, PyAny>, diff --git a/fensu.toml b/fensu.toml index 88cb94cfe6..26a7e3dc38 100644 --- a/fensu.toml +++ b/fensu.toml @@ -3635,7 +3635,7 @@ reason = "Each registers its own module's Python functions; only the add-functio [[dupes.allowlist]] paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs"] -reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." +reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs"] @@ -3673,10 +3673,6 @@ reason = "Each registers its own module's Python functions; only the add-functio paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/getters.rs"] reason = "Each matches its own native class's attribute names to its own fields; only the cast-and-dispatch shape is alike." -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/getters.rs"] -reason = "Each matches its own native class's attribute names to its own fields; only the cast-and-dispatch shape is alike." - [[dupes.allowlist]] paths = ["crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs"] reason = "Each writes the entries of its own plan section with its own keys; only mapping over the entries is alike." diff --git a/tests/native_field_fallbacks.txt b/tests/native_field_fallbacks.txt index 3be8268760..08510b752e 100644 --- a/tests/native_field_fallbacks.txt +++ b/tests/native_field_fallbacks.txt @@ -326,8 +326,8 @@ crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs MigrationDecision.moves_data crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs PlannerRelationsContext.source_warehouse_columns crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs FutureCursorAction.value -crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs CursorOverrides.typed_as -crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs DbtCompileReferenceResolver.resolve_reference +crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs CursorOverrides.typed_as +crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs DbtCompileReferenceResolver.resolve_reference crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs ScenarioArtifactKind.value crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs PlannerRelationsContext.model_locations crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs PlannerRelationsContext.model_locations