diff --git a/Cargo.lock b/Cargo.lock index 1e5bb057d2..5887df3588 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4450,7 +4450,6 @@ dependencies = [ "sqlbuild-cli", "sqlbuild-config", "sqlbuild-core", - "sqlbuild-cursor", "sqlbuild-discovery", "sqlbuild-host", "sqlbuild-model-config", @@ -4459,6 +4458,7 @@ dependencies = [ "sqlbuild-pysource", "sqlbuild-python-base", "sqlbuild-python-front", + "sqlbuild-python-planner", "sqlbuild-python-project", "sqlbuild-recall", "sqlbuild-refactor", @@ -4468,7 +4468,6 @@ dependencies = [ "sqlbuild-sqltext", "sqlbuild-state", "sqlclean", - "uuid", ] [[package]] @@ -4525,6 +4524,29 @@ dependencies = [ "uuid", ] +[[package]] +name = "sqlbuild-python-planner" +version = "0.0.0" +dependencies = [ + "pyo3", + "serde_json", + "sqlbuild-analysis", + "sqlbuild-cache", + "sqlbuild-core", + "sqlbuild-cursor", + "sqlbuild-discovery", + "sqlbuild-host", + "sqlbuild-model-config", + "sqlbuild-project", + "sqlbuild-python-base", + "sqlbuild-python-front", + "sqlbuild-python-project", + "sqlbuild-rules", + "sqlbuild-sqltext", + "sqlbuild-state", + "uuid", +] + [[package]] name = "sqlbuild-python-project" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 81686661c9..1af3b2707f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ members = [ "crates/sqlbuild-python-base", "crates/sqlbuild-python-front", "crates/sqlbuild-python-project", + "crates/sqlbuild-python-planner", "crates/sqlbuild-python", ] resolver = "2" @@ -61,9 +62,10 @@ order = [ "sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", + "sqlbuild-python-planner", "sqlbuild-python", ] -python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python"] +python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python-planner", "sqlbuild-python"] polyglot-floor = "sqlbuild-analysis" [workspace.package] @@ -186,6 +188,7 @@ sqlbuild-project = { path = "crates/sqlbuild-project" } sqlbuild-python-base = { path = "crates/sqlbuild-python-base" } sqlbuild-python-front = { path = "crates/sqlbuild-python-front" } sqlbuild-python-project = { path = "crates/sqlbuild-python-project" } +sqlbuild-python-planner = { path = "crates/sqlbuild-python-planner" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } sqlbuild-recall = { path = "crates/sqlbuild-recall" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } diff --git a/crates/README.md b/crates/README.md index d6edeeaef9..5c7d6d822c 100644 --- a/crates/README.md +++ b/crates/README.md @@ -31,14 +31,15 @@ tested and benchmarked with `cargo test` alone. | `sqlbuild-python-base` | The base of the PyO3 bindings: native record, enum and error classes with their specs, registration, construction and field reads; keyword calls, imports, conversions and panics at the Python boundary; host entry calls; the process allocator; and the shared runtime the binding modules build on: the native store, compile context, diagnostics, progress and terminal text, warehouse access and orchestration reads. | | `sqlbuild-python-front` | The front of the compile pipeline as PyO3 bindings, on top of `sqlbuild-python-base`: project discovery, model configuration and headers, scopes, macro rendering and authored values, SQL text, model attachment and the compile driver, cursors, contract and refactor facts, and the warehouse state tables of consumption, observations and audit results. | | `sqlbuild-python-project` | The compiled project as PyO3 bindings, on top of `sqlbuild-python-front`: the project host and the Python views of compiled models, inputs and resources, the compiled-project snapshot, analysis classes and the analysis session, the project graph, references, lineage, fingerprints, contracts and semantic shapes, audit rendering, and the planner's identities, scope, selection, phases, fixture strategy and SQL test planning. | -| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. | +| `sqlbuild-python-planner` | The plan layer as PyO3 bindings, on top of `sqlbuild-python-project`: execution plans and their entries, planner phases, change detection, migrations, reference and cursor resolution, scenarios, strategy and warehouse inspection, plus freshness, consumption, microbatches, observations, lifecycle events and plan output. | +| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-planner`, `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. | ## Layering Dependencies point one way, from the top of this graph to the bottom: ```text -sqlbuild-python -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base +sqlbuild-python -> sqlbuild-python-planner -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base -> sqlbuild-project -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core diff --git a/crates/sqlbuild-python-front/src/cost_context/_helpers/context.rs b/crates/sqlbuild-python-front/src/cost_context/_helpers/context.rs new file mode 100644 index 0000000000..561f4bb9b0 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/_helpers/context.rs @@ -0,0 +1,48 @@ +//! The `ContextVar` that holds the current cost resource context, and the scopes built over it. + +use crate::cost_context::main::process_object::process_object; +use crate::cost_context::models::CostScope; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyDict; + +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +const CONTEXT_VARIABLE_NAME: &str = "sqlbuild_cost_resource_context"; +pub(crate) const DEFAULT_PHASE: &str = "execute"; +pub(crate) const DEFAULT_ATTEMPT: i64 = 1; + +static CURRENT: PyOnceLock> = PyOnceLock::new(); + +/// The `ContextVar` holding the current cost resource context. +pub(crate) fn current_variable(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + process_object(&CURRENT, py, || { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("default", py.None())?; + imported(py, "contextvars", "ContextVar")?.call((CONTEXT_VARIABLE_NAME,), Some(&options)) + }) +} + +pub(crate) fn or_current<'py>( + value: Bound<'py, PyAny>, + current: &Bound<'py, PyAny>, + field: &str, +) -> PyResult> { + if !value.is_none() || current.is_none() { + return Ok(value); + } + native_field(current, field) +} + +pub(crate) fn scope_of(context: Bound<'_, PyAny>) -> CostScope { + CostScope { + context: Some(context.unbind()), + token: None, + } +} + +/// The active `CostResourceContext`, or `None`. +pub(crate) fn current_context(py: Python<'_>) -> PyResult> { + current_variable(py)?.call_method0("get") +} diff --git a/crates/sqlbuild-python-front/src/cost_context/_helpers/mod.rs b/crates/sqlbuild-python-front/src/cost_context/_helpers/mod.rs new file mode 100644 index 0000000000..e3c01fd8bd --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod context; diff --git a/crates/sqlbuild-python-front/src/cost_context/constants.rs b/crates/sqlbuild-python-front/src/cost_context/constants.rs new file mode 100644 index 0000000000..aca88d3b23 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/constants.rs @@ -0,0 +1,6 @@ +//! Constants of cost context. + +/// The methods an adapter with warehouse cost telemetry implements. +pub(crate) const COST_AWARE_METHODS: [&str; 2] = ["cost_capability", "collect_run_cost"]; + +pub(crate) const DECIMAL_MODULE: &str = "decimal"; diff --git a/crates/sqlbuild-python-front/src/cost_context/main/cost_aware.rs b/crates/sqlbuild-python-front/src/cost_context/main/cost_aware.rs new file mode 100644 index 0000000000..9a78915d42 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/main/cost_aware.rs @@ -0,0 +1,15 @@ +//! Whether an adapter implements the cost telemetry methods. + +use crate::cost_context::constants::COST_AWARE_METHODS; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Whether `adapter` implements the cost telemetry methods. +pub fn cost_aware(adapter: &Bound<'_, PyAny>) -> PyResult { + for name in COST_AWARE_METHODS { + if !adapter.hasattr(name)? || native_field(adapter, name)?.is_none() { + return Ok(false); + } + } + Ok(true) +} diff --git a/crates/sqlbuild-python-front/src/cost_context/main/instant_micros.rs b/crates/sqlbuild-python-front/src/cost_context/main/instant_micros.rs new file mode 100644 index 0000000000..215d1ec7fb --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/main/instant_micros.rs @@ -0,0 +1,25 @@ +//! Microseconds since the epoch: UTC for aware values, the wall clock for naive ones. + +use crate::cursor::constants::DATETIME_MODULE; +use crate::state_rows::main::event_rows::offset_aware::offset_aware; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::main::imported::imported; + +/// Microseconds since the epoch: UTC for aware values, the wall clock for naive ones. +pub fn instant_micros(value: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = value.py(); + let datetime: Bound<'_, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; + let epoch: Bound<'_, PyAny> = if offset_aware(value)? { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("tzinfo", imported(py, DATETIME_MODULE, "UTC")?)?; + datetime.call((1970, 1, 1), Some(&options))? + } else { + datetime.call1((1970, 1, 1))? + }; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("microseconds", 1)?; + let micro: Bound<'_, PyAny> = + imported(py, DATETIME_MODULE, "timedelta")?.call((), Some(&options))?; + value.sub(epoch)?.floor_div(micro)?.extract() +} diff --git a/crates/sqlbuild-python-front/src/cost_context/main/mod.rs b/crates/sqlbuild-python-front/src/cost_context/main/mod.rs new file mode 100644 index 0000000000..48557d16dd --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/main/mod.rs @@ -0,0 +1,6 @@ +pub mod cost_aware; +pub mod instant_micros; +pub mod native_decimal; +pub mod process_object; +pub mod python_decimal; +pub mod register_cost_context; diff --git a/crates/sqlbuild-python-front/src/cost_context/main/native_decimal.rs b/crates/sqlbuild-python-front/src/cost_context/main/native_decimal.rs new file mode 100644 index 0000000000..9c56b9ce38 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/main/native_decimal.rs @@ -0,0 +1,24 @@ +//! A finite non-negative `Decimal` as its exact digits and exponent. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::cost_allocation::models::PyDecimal; + +/// A finite non-negative `Decimal` as its exact digits and exponent. +pub fn native_decimal(value: &Bound<'_, PyAny>) -> PyResult { + let (sign, digits, exponent): (u8, Vec, Bound<'_, PyAny>) = + value.call_method0("as_tuple")?.extract()?; + let Ok(exponent) = exponent.extract::() else { + return Err(PyValueError::new_err("cost decimals must be finite")); + }; + if sign != 0 && digits.iter().any(|digit| *digit != 0) { + return Err(PyValueError::new_err("cost decimals must not be negative")); + } + let leading: usize = digits.iter().take_while(|digit| **digit == 0).count(); + let digits: Vec = if leading == digits.len() { + vec![0] + } else { + digits[leading..].to_vec() + }; + Ok(PyDecimal { digits, exponent }) +} diff --git a/crates/sqlbuild-python-front/src/cost_context/main/process_object.rs b/crates/sqlbuild-python-front/src/cost_context/main/process_object.rs new file mode 100644 index 0000000000..22fbe5b302 --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/main/process_object.rs @@ -0,0 +1,14 @@ +//! The process-wide Python object in a cell, built on first use. + +use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyResult, Python}; +use pyo3::sync::PyOnceLock; + +/// The process-wide Python object in `cell`, built by `build` on first use. +pub fn process_object<'py>( + cell: &'static PyOnceLock>, + py: Python<'py>, + build: impl FnOnce() -> PyResult>, +) -> PyResult<&'py Bound<'py, PyAny>> { + cell.get_or_try_init(py, || Ok::, PyErr>(build()?.unbind())) + .map(|object| object.bind(py)) +} diff --git a/crates/sqlbuild-python-front/src/cost_context/main/python_decimal.rs b/crates/sqlbuild-python-front/src/cost_context/main/python_decimal.rs new file mode 100644 index 0000000000..6639d742ee --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/main/python_decimal.rs @@ -0,0 +1,21 @@ +//! `Decimal((0, digits, exponent))`. + +use crate::cost_context::constants::DECIMAL_MODULE; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_core::cost_allocation::models::PyDecimal; +use sqlbuild_python_base::boundary::main::imported::imported; + +/// `Decimal((0, digits, exponent))`. +pub fn python_decimal<'py>(py: Python<'py>, value: &PyDecimal) -> PyResult> { + let parts: Bound<'py, PyTuple> = PyTuple::new( + py, + [ + 0_i64.into_bound_py_any(py)?, + PyTuple::new(py, &value.digits)?.into_any(), + value.exponent.into_bound_py_any(py)?, + ], + )?; + imported(py, DECIMAL_MODULE, "Decimal")?.call1((parts,)) +} diff --git a/crates/sqlbuild-python-front/src/cost_context/main/register_cost_context.rs b/crates/sqlbuild-python-front/src/cost_context/main/register_cost_context.rs new file mode 100644 index 0000000000..e9f465956b --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/main/register_cost_context.rs @@ -0,0 +1,11 @@ +//! Register the cost context classes. + +use crate::cost_context::models::{CostContext, CostScope}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the cost context classes. +pub fn register_cost_context(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python-front/src/cost_context/mod.rs b/crates/sqlbuild-python-front/src/cost_context/mod.rs new file mode 100644 index 0000000000..3e4f4f853a --- /dev/null +++ b/crates/sqlbuild-python-front/src/cost_context/mod.rs @@ -0,0 +1,6 @@ +//! The per-resource cost attribution context and the Python cost values it reads and builds. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs b/crates/sqlbuild-python-front/src/cost_context/models.rs similarity index 72% rename from crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs rename to crates/sqlbuild-python-front/src/cost_context/models.rs index 2dfcc4d08d..9c623534a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs +++ b/crates/sqlbuild-python-front/src/cost_context/models.rs @@ -1,42 +1,23 @@ -//! `CostContext`: the per-resource cost attribution context, held in a Python `ContextVar`. +//! Public records of cost context. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; +use crate::cost_context::_helpers::context::{ + DEFAULT_ATTEMPT, DEFAULT_PHASE, current_context, current_variable, or_current, scope_of, +}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple}; use pyo3::{IntoPyObjectExt, pyclass, pymethods}; use sqlbuild_host::entries::models::ExecutorHost; - -use crate::bindings::_helpers::cost_telemetry::values::process_object; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; -const CONTEXT_VARIABLE_NAME: &str = "sqlbuild_cost_resource_context"; -const DEFAULT_PHASE: &str = "execute"; -const DEFAULT_ATTEMPT: i64 = 1; - -static CURRENT: PyOnceLock> = PyOnceLock::new(); - -fn current_variable(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { - process_object(&CURRENT, py, || { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("default", py.None())?; - imported(py, "contextvars", "ContextVar")?.call((CONTEXT_VARIABLE_NAME,), Some(&options)) - }) -} - -/// The active `CostResourceContext`, or `None`. -pub(crate) fn current_context(py: Python<'_>) -> PyResult> { - current_variable(py)?.call_method0("get") -} - /// A `with` scope that makes one resource context current, or does nothing without one. #[pyclass(module = "sqlbuild._native")] -pub(crate) struct CostScope { - context: Option>, - token: Option>, +#[derive(Debug)] +pub struct CostScope { + pub(crate) context: Option>, + pub(crate) token: Option>, } #[pymethods] @@ -61,27 +42,10 @@ impl CostScope { } } -fn or_current<'py>( - value: Bound<'py, PyAny>, - current: &Bound<'py, PyAny>, - field: &str, -) -> PyResult> { - if !value.is_none() || current.is_none() { - return Ok(value); - } - native_field(current, field) -} - -fn scope_of(context: Bound<'_, PyAny>) -> CostScope { - CostScope { - context: Some(context.unbind()), - token: None, - } -} - /// Run-cost execution context entry points. #[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct CostContext; +#[derive(Debug)] +pub struct CostContext; #[pymethods] impl CostContext { @@ -92,7 +56,7 @@ impl CostContext { #[staticmethod] #[pyo3(signature = (**options))] - pub(crate) fn scope(py: Python<'_>, options: Option>) -> PyResult { + pub fn scope(py: Python<'_>, options: Option>) -> PyResult { let arguments: KeywordArguments<'_> = KeywordArguments::read( py, "scope", @@ -139,7 +103,7 @@ impl CostContext { #[staticmethod] #[pyo3(signature = (**options))] - pub(crate) fn resource_scope( + pub fn resource_scope( py: Python<'_>, options: Option>, ) -> PyResult { @@ -187,9 +151,3 @@ impl CostContext { Ok(scope_of(context)) } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/lib.rs b/crates/sqlbuild-python-front/src/lib.rs index b0070dbde8..183d051c5e 100644 --- a/crates/sqlbuild-python-front/src/lib.rs +++ b/crates/sqlbuild-python-front/src/lib.rs @@ -8,6 +8,7 @@ pub mod command_runtime; pub mod compile_attachment; pub mod compile_driver; pub mod contracts; +pub mod cost_context; pub mod cursor; pub mod discovery; pub mod macro_bridge; diff --git a/crates/sqlbuild-python-planner/Cargo.toml b/crates/sqlbuild-python-planner/Cargo.toml new file mode 100644 index 0000000000..2495999310 --- /dev/null +++ b/crates/sqlbuild-python-planner/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "sqlbuild-python-planner" +description = "The plan layer of the sqlbuild._native bindings: execution plans and their entries, planner phases, changes, migrations, resolution, scenarios, strategy and warehouse state, freshness, consumption, microbatches, observations, lifecycle events and plan output." +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +pyo3.workspace = true +serde_json.workspace = true +sqlbuild-analysis.workspace = true +sqlbuild-cache.workspace = true +sqlbuild-core.workspace = true +sqlbuild-cursor.workspace = true +sqlbuild-discovery.workspace = true +sqlbuild-host.workspace = true +sqlbuild-model-config.workspace = true +sqlbuild-project.workspace = true +sqlbuild-python-base.workspace = true +sqlbuild-python-front.workspace = true +sqlbuild-python-project.workspace = true +sqlbuild-rules.workspace = true +sqlbuild-sqltext.workspace = true +sqlbuild-state.workspace = true +uuid.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/facts.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs rename to crates/sqlbuild-python-planner/src/consumption/_helpers/facts.rs index ebd65842a7..ac4fe2ff2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs +++ b/crates/sqlbuild-python-planner/src/consumption/_helpers/facts.rs @@ -1,5 +1,6 @@ //! The facts `--changed` and the consumption recorder read from the plan and compiled project. +use crate::consumption::main::relation::relation; use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -7,13 +8,11 @@ use pyo3::types::PyDict; use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; use sqlbuild_state::consumption::models::{ForcedRun, InputFact, NodeFacts, NodeKey, RelationKey}; -use crate::bindings::_helpers::consumption::relation_reads::{ - Locations, relation, sql_reads_undeclared, -}; -use crate::bindings::_helpers::planner_identity::identities::hook_version_hashes; -use crate::bindings::_helpers::planner_identity::metadata::{ - function_local_hashes, model_version_identity_metadata_json, seed_identity, -}; +use crate::consumption::_helpers::relation_reads::{Locations, sql_reads_undeclared}; +use crate::planner_identity::main::function_local_hashes::function_local_hashes; +use crate::planner_identity::main::hook_version_hashes::hook_version_hashes; +use crate::planner_identity::main::model_version_identity_metadata_json::model_version_identity_metadata_json; +use crate::planner_identity::main::seed_identity::seed_identity; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_project::compiled_views::main::compiled_models::dep_rows::dep_rows; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/mod.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/consumption/mod.rs rename to crates/sqlbuild-python-planner/src/consumption/_helpers/mod.rs diff --git a/crates/sqlbuild-python-planner/src/consumption/_helpers/policy.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/policy.rs new file mode 100644 index 0000000000..940a0e48ab --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/_helpers/policy.rs @@ -0,0 +1,43 @@ +//! What `--changed` counts and how far a change reaches: the command line, else `[changes]`. + +use std::collections::BTreeSet; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_state::consumption::models::{ChangeKind, NodeKey, TriggeredJob}; + +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +pub(crate) const DEFINITION: &str = "definition"; +pub(crate) const SOURCE_DATA: &str = "source_data"; +pub(crate) const REACH_JOB: &str = "job"; +pub(crate) const REACH_ALL: &str = "all"; + +pub(crate) fn change_kind(name: &str) -> PyResult { + match name { + DEFINITION => Ok(ChangeKind::Definition), + SOURCE_DATA => Ok(ChangeKind::SourceData), + other => Err(PyValueError::new_err(format!( + "unknown change kind '{other}': expected {DEFINITION} or {SOURCE_DATA}" + ))), + } +} + +/// The request's `--select` selectors. +pub(crate) fn selectors(request: &Bound<'_, PyAny>) -> PyResult> { + Ok(native_field(request, "select")? + .extract::>>()? + .unwrap_or_default()) +} + +/// A triggered job with its selected nodes as consumption keys. +pub(crate) fn triggered_job((name, keys): (String, BTreeSet)) -> TriggeredJob { + TriggeredJob { + name, + nodes: keys + .into_iter() + .map(|(kind, name)| NodeKey { kind, name }) + .collect(), + } +} diff --git a/crates/sqlbuild-python-planner/src/consumption/_helpers/recorder.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/recorder.rs new file mode 100644 index 0000000000..9afb032387 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/_helpers/recorder.rs @@ -0,0 +1,268 @@ +//! Record what each model and seed execution consumed, and decide `--changed` skips, around one build. + +use crate::consumption::models::ChangedDecision; +use crate::consumption::models::ConsumptionRecorder; +use crate::consumption::models::Prepared; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_state::consumption::constants::RECENT_EXECUTIONS; +use sqlbuild_state::consumption::main::changed_run_set::changed_run_set; +use sqlbuild_state::consumption::main::consumption_history::consumption_history; +use sqlbuild_state::consumption::main::read_back_history::read_back_history; +use sqlbuild_state::consumption::main::started_append::started_append; +use sqlbuild_state::consumption::models::{ + ChangedOutcome, ChangedPolicy, ConsumptionAppend, ConsumptionHistory, ConsumptionRow, HeldBack, + InputFact, NodeFacts, NodeKey, Reach, RelationKey, RunReason, +}; +use uuid::Uuid; + +use crate::consumption::_helpers::facts::{BuildFacts, MODEL, SEED, build_facts}; +use sqlbuild_python_front::native_host::models::{ + ConsumptionConnection, Location, NodeCompleteHook, +}; + +/// Succeeded rows are flushed once this many executions are pending. +pub(crate) const FLUSH_EXECUTIONS: usize = 100; +/// Or once this long has passed since the last flush. +pub(crate) const FLUSH_INTERVAL: Duration = Duration::from_secs(30); +const MODEL_RESULT: &str = "model_name"; +const SEED_RESULT: &str = "seed_name"; +pub(crate) const SUCCESS: &str = "success"; +pub(crate) const ERROR_OUTCOME: &str = "error"; + +/// The mutable side of one build's recording. +#[derive(Debug)] +pub(crate) struct Recording { + pub(crate) pending: BTreeMap>, + pub(crate) pending_executions: usize, + pub(crate) pending_seeds: Vec, + pub(crate) tables: BTreeSet, + pub(crate) last_flush: Instant, + pub(crate) warnings: Vec, +} + +impl Default for Recording { + fn default() -> Self { + Self { + pending: BTreeMap::new(), + pending_executions: 0, + pending_seeds: Vec::new(), + tables: BTreeSet::new(), + last_flush: Instant::now(), + warnings: Vec::new(), + } + } +} + +pub(crate) fn kind_of(result: &Bound<'_, PyAny>) -> PyResult> { + for (field, kind) in [(MODEL_RESULT, MODEL), (SEED_RESULT, SEED)] { + if let Some(name) = result.getattr_opt(field)? { + return Ok(Some((kind, name.extract()?))); + } + } + Ok(None) +} + +/// Schemas to read records from: all under `--changed`, else those of unselected upstreams. +fn read_locations(facts: &BuildFacts, changed: bool) -> BTreeSet { + let by_key: HashMap<&NodeKey, &NodeFacts> = + facts.nodes.iter().map(|node| (&node.key, node)).collect(); + if changed { + return facts + .nodes + .iter() + .map(|node| Location::of(&node.relation)) + .collect(); + } + facts + .nodes + .iter() + .filter(|node| facts.runnable.contains(&node.key)) + .flat_map(|node| node.inputs.iter()) + .filter_map(|input| match input { + InputFact::Node(upstream) if !facts.runnable.contains(upstream) => by_key.get(upstream), + _ => None, + }) + .map(|node| Location::of(&node.relation)) + .collect() +} + +/// Read records, decide `--changed` skips, mint execution IDs and append `started` rows. +pub(crate) fn prepare<'py>( + (plan, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter, connection_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (policy, full_refresh): (Option<&ChangedPolicy>, bool), + (run_id, base_on_node_complete): (String, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = plan.py(); + let facts: BuildFacts = build_facts((plan, project), (policy.is_some(), full_refresh))?; + let connection = ConsumptionConnection::new(adapter, connection_config); + let (history, warnings): (ConsumptionHistory, Vec) = + read_history(py, &connection, (&facts, policy)); + let decision: Option = + policy.map(|policy| changed_decision(&facts, &history, policy)); + let not_run: BTreeSet<&NodeKey> = decision.iter().flat_map(ChangedDecision::not_run).collect(); + let executions: HashMap = facts + .runnable + .iter() + .filter(|key| !not_run.contains(key)) + .map(|key| (key.clone(), Uuid::now_v7().to_string())) + .collect(); + let nodes: HashMap = facts + .nodes + .into_iter() + .map(|node| (node.key.clone(), node)) + .collect(); + let mut started: BTreeMap> = BTreeMap::new(); + for (key, execution_id) in &executions { + if let Some(node) = nodes.get(key) { + started + .entry(Location::of(&node.relation)) + .or_default() + .push(started_append(node, execution_id, &run_id)); + } + } + let mut failed_start: HashMap = HashMap::new(); + let mut tables: BTreeSet = BTreeSet::new(); + for (location, rows) in started { + let written: PyResult<()> = connection + .ensure_table(py, &location) + .and_then(|()| connection.append(py, &location, &rows)); + match written { + Ok(()) => { + tables.insert(location); + } + Err(error) => { + let message: String = error.to_string(); + for row in rows { + failed_start.insert(row.node, message.clone()); + } + } + } + } + connection.close(py)?; + let recorder = ConsumptionRecorder { + connection, + run_id, + nodes, + history, + executions, + failed_start, + base_on_node_complete: NodeCompleteHook::new(&base_on_node_complete), + recording: Mutex::new(Recording { + tables, + ..Recording::default() + }), + }; + Ok(Prepared { + recorder: Bound::new(py, recorder)?, + decision, + warnings, + }) +} + +/// Recent executions the build reads, with older ones read back when deciding needs them. +pub(crate) fn read_history( + py: Python<'_>, + connection: &ConsumptionConnection, + (facts, policy): (&BuildFacts, Option<&ChangedPolicy>), +) -> (ConsumptionHistory, Vec) { + let mut warnings: Vec = Vec::new(); + let mut rows: Vec = Vec::new(); + for location in read_locations(facts, policy.is_some()) { + match connection.read_recent(py, &location) { + Ok(read) => rows.extend(read), + Err(error) => warnings.push(format!( + "could not read consumed inputs in schema {}: {error}; its models and seeds count as changed", + location.schema + )), + } + } + let history: ConsumptionHistory = consumption_history(&rows, RECENT_EXECUTIONS); + if policy.is_some_and(needs_read_back) { + let (history, read_back): (ConsumptionHistory, Vec) = + with_read_back(py, connection, history, &facts.nodes); + warnings.extend(read_back); + return (history, warnings); + } + (history, warnings) +} + +/// What `--changed` does with each runnable node. +pub(crate) fn changed_decision( + facts: &BuildFacts, + history: &ConsumptionHistory, + policy: &ChangedPolicy, +) -> ChangedDecision { + let mut reasons: Vec<(NodeKey, RunReason)> = Vec::new(); + let mut skipped: Vec = Vec::new(); + let mut held_back: Vec<(NodeKey, HeldBack)> = Vec::new(); + for (key, outcome) in changed_run_set(&facts.nodes, history, policy) { + if !facts.runnable.contains(&key) { + continue; + } + match outcome { + ChangedOutcome::Runs(reason) => reasons.push((key, reason)), + ChangedOutcome::Unchanged => skipped.push(key), + ChangedOutcome::HeldBack(why) => held_back.push((key, why)), + } + } + ChangedDecision { + policy: policy.clone(), + run: reasons.len(), + skipped, + held_back, + reasons, + } +} + +/// Whether deciding needs older executions: to tell definition from data, or to attribute a change. +fn needs_read_back(policy: &ChangedPolicy) -> bool { + policy.counts.len() == 1 || matches!(policy.reach, Reach::Job(_)) +} + +/// `history` plus every older upstream execution its builds read, followed back until none is left. +fn with_read_back( + py: Python<'_>, + connection: &ConsumptionConnection, + history: ConsumptionHistory, + nodes: &[NodeFacts], +) -> (ConsumptionHistory, Vec) { + let mut warnings: BTreeSet = BTreeSet::new(); + let history: ConsumptionHistory = read_back_history(history, nodes, |references| { + let (rows, failures) = read_references(py, connection, references); + warnings.extend(failures); + rows + }); + (history, warnings.into_iter().collect()) +} + +/// The rows of these executions; a schema that cannot be read leaves its rebuilds counted. +fn read_references( + py: Python<'_>, + connection: &ConsumptionConnection, + references: &[(RelationKey, String)], +) -> (Vec, Vec) { + let mut by_location: BTreeMap> = BTreeMap::new(); + for (relation, id) in references { + by_location + .entry(Location::of(relation)) + .or_default() + .push(id.as_str()); + } + let mut rows: Vec = Vec::new(); + let mut warnings: Vec = Vec::new(); + for (location, ids) in by_location { + match connection.read_executions(py, &location, &ids) { + Ok(read) => rows.extend(read), + Err(error) => warnings.push(format!( + "could not read earlier executions in schema {}: {error}; rebuilt upstreams there count as changed", + location.schema + )), + } + } + (rows, warnings) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs b/crates/sqlbuild-python-planner/src/consumption/_helpers/relation_reads.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs rename to crates/sqlbuild-python-planner/src/consumption/_helpers/relation_reads.rs index d5f910f5d1..25bb384cb0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs +++ b/crates/sqlbuild-python-planner/src/consumption/_helpers/relation_reads.rs @@ -23,15 +23,6 @@ pub(crate) struct Locations { by_name: HashMap>, } -/// The relation a model or seed destination names; a destination without a schema has an empty one. -pub(crate) fn relation(destination: &Bound<'_, PyAny>) -> PyResult { - Ok(RelationKey { - database: optional_text(&native_field(destination, "database")?)?, - schema: optional_text(&native_field(destination, "schema")?)?.unwrap_or_default(), - name: native_field(destination, "name")?.extract()?, - }) -} - impl Locations { /// `nodes` holds every model and seed relation of the project. pub(crate) fn read( diff --git a/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs b/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs new file mode 100644 index 0000000000..ca236b42d0 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs @@ -0,0 +1,61 @@ +//! The policy of a `--changed` request, or None without it; triggered jobs resolve on `project`. + +use crate::consumption::_helpers::policy::DEFINITION; +use crate::consumption::_helpers::policy::REACH_ALL; +use crate::consumption::_helpers::policy::REACH_JOB; +use crate::consumption::_helpers::policy::change_kind; +use crate::consumption::_helpers::policy::selectors; +use crate::consumption::_helpers::policy::triggered_job; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; +use sqlbuild_python_project::compiled_project::main::triggered_jobs::triggered_jobs; +use sqlbuild_python_project::graph::models::NativeProjectGraph; +use sqlbuild_state::consumption::models::{ChangeKind, ChangedPolicy, Reach}; +use std::collections::BTreeSet; + +/// The policy of a `--changed` request, or None without it; triggered jobs resolve on `project`. +pub fn changed_policy( + project_dir: &Bound<'_, PyAny>, + request: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, +) -> PyResult> { + if !native_field(request, "changed")?.is_truthy()? { + return Ok(None); + } + let configured = configured_changes(project_dir)?; + let counts: Vec = match native_field(request, "changed_counts")?.extract()? { + Some(counts) => counts, + None => configured + .counts + .unwrap_or_else(|| vec![DEFINITION.to_owned()]), + }; + let reach: String = match native_field(request, "reach")?.extract()? { + Some(reach) => reach, + None => configured.reach.unwrap_or_else(|| REACH_JOB.to_owned()), + }; + let counts: BTreeSet = counts + .iter() + .map(|name| change_kind(name)) + .collect::>()?; + let reach: Reach = match reach.as_str() { + REACH_ALL => Reach::All, + REACH_JOB => { + let graph: PyRef<'_, NativeProjectGraph> = + native_field(project, "lineage_graph")?.extract()?; + Reach::Job( + triggered_jobs(project.py(), graph.graph(), &selectors(request)?)? + .into_iter() + .map(triggered_job) + .collect(), + ) + } + other => { + return Err(PyValueError::new_err(format!( + "unknown reach '{other}': expected {REACH_JOB} or {REACH_ALL}" + ))); + } + }; + Ok(Some(ChangedPolicy { counts, reach })) +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/held_back_text.rs b/crates/sqlbuild-python-planner/src/consumption/main/held_back_text.rs new file mode 100644 index 0000000000..3b03a2bb96 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/held_back_text.rs @@ -0,0 +1,14 @@ +//! Why `--changed` holds a node back, as one line for verbose output. + +use sqlbuild_state::consumption::models::HeldBack; + +/// Why `--changed` holds a node back, as one line for verbose output. +pub fn held_back_text(why: &HeldBack) -> String { + match why { + HeldBack::RefreshedBy(jobs) => format!( + "job {} refreshes it, and the change is not in that job", + jobs.join(", ") + ), + HeldBack::ReadsHeldBack(key) => format!("reads held-back {} {}", key.kind, key.name), + } +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/kind_value.rs b/crates/sqlbuild-python-planner/src/consumption/main/kind_value.rs new file mode 100644 index 0000000000..70d7c85ee5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/kind_value.rs @@ -0,0 +1,13 @@ +//! A change kind as `[changes] counts` and `--changed=` spell it. + +use crate::consumption::_helpers::policy::DEFINITION; +use crate::consumption::_helpers::policy::SOURCE_DATA; +use sqlbuild_state::consumption::models::ChangeKind; + +/// A change kind as `[changes] counts` and `--changed=` spell it. +pub const fn kind_value(kind: ChangeKind) -> &'static str { + match kind { + ChangeKind::Definition => DEFINITION, + ChangeKind::SourceData => SOURCE_DATA, + } +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/mod.rs b/crates/sqlbuild-python-planner/src/consumption/main/mod.rs new file mode 100644 index 0000000000..3f9152ccc5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/mod.rs @@ -0,0 +1,9 @@ +pub mod changed_policy; +pub mod held_back_text; +pub mod kind_value; +pub mod policy_line; +pub mod prepare; +pub mod preview_changed; +pub mod reach_value; +pub mod relation; +pub mod run_reason_text; diff --git a/crates/sqlbuild-python-planner/src/consumption/main/policy_line.rs b/crates/sqlbuild-python-planner/src/consumption/main/policy_line.rs new file mode 100644 index 0000000000..bb5e3f5068 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/policy_line.rs @@ -0,0 +1,24 @@ +//! One line saying what the policy counts and how far it reaches, as build and plan print it. + +use crate::consumption::main::reach_value::reach_value; +use sqlbuild_state::consumption::models::{ChangeKind, ChangedPolicy}; + +/// One line saying what the policy counts and how far it reaches, as build and plan print it. +pub fn policy_line(policy: &ChangedPolicy) -> String { + let label = |kind: &ChangeKind| match kind { + ChangeKind::Definition => "definition", + ChangeKind::SourceData => "source data", + }; + let counted: Vec<&str> = policy.counts.iter().map(label).collect(); + let uncounted: Vec = [ChangeKind::Definition, ChangeKind::SourceData] + .iter() + .filter(|kind| !policy.counts.contains(kind)) + .map(|kind| format!(" ({} not counted)", label(kind))) + .collect(); + format!( + "Counting: {}{} ยท reach: {}", + counted.join(", "), + uncounted.concat(), + reach_value(&policy.reach) + ) +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/prepare.rs b/crates/sqlbuild-python-planner/src/consumption/main/prepare.rs new file mode 100644 index 0000000000..c80d70d8a4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/prepare.rs @@ -0,0 +1,22 @@ +//! Read records, decide `--changed` skips, mint execution IDs and append `started` rows. + +use crate::consumption::_helpers::recorder; +use crate::consumption::models::Prepared; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_state::consumption::models::ChangedPolicy; + +/// Read records, decide `--changed` skips, mint execution IDs and append `started` rows. +#[inline] +pub fn prepare<'py>( + (plan, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter, connection_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (policy, full_refresh): (Option<&ChangedPolicy>, bool), + (run_id, base_on_node_complete): (String, Bound<'py, PyAny>), +) -> PyResult> { + recorder::prepare( + (plan, project), + (adapter, connection_config), + (policy, full_refresh), + (run_id, base_on_node_complete), + ) +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/preview_changed.rs b/crates/sqlbuild-python-planner/src/consumption/main/preview_changed.rs new file mode 100644 index 0000000000..0f5cb9af37 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/preview_changed.rs @@ -0,0 +1,24 @@ +//! What `plan --changed` previews: what a build would run now, read without recording anything. + +use crate::consumption::_helpers::facts::{BuildFacts, build_facts}; +use crate::consumption::_helpers::recorder::changed_decision; +use crate::consumption::_helpers::recorder::read_history; +use crate::consumption::models::ChangedDecision; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_front::native_host::models::ConsumptionConnection; +use sqlbuild_state::consumption::models::{ChangedPolicy, ConsumptionHistory}; + +/// What `plan --changed` previews: what a build would run now, read without recording anything. +pub fn preview_changed( + (plan, project): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (policy, full_refresh): (&ChangedPolicy, bool), +) -> PyResult<(ChangedDecision, Vec)> { + let py: Python<'_> = plan.py(); + let facts: BuildFacts = build_facts((plan, project), (true, full_refresh))?; + let connection = ConsumptionConnection::new(adapter, connection_config); + let (history, warnings): (ConsumptionHistory, Vec) = + read_history(py, &connection, (&facts, Some(policy))); + connection.close(py)?; + Ok((changed_decision(&facts, &history, policy), warnings)) +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/reach_value.rs b/crates/sqlbuild-python-planner/src/consumption/main/reach_value.rs new file mode 100644 index 0000000000..e00f3477db --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/reach_value.rs @@ -0,0 +1,13 @@ +//! A reach as `[changes] reach` and `--reach` spell it. + +use crate::consumption::_helpers::policy::REACH_ALL; +use crate::consumption::_helpers::policy::REACH_JOB; +use sqlbuild_state::consumption::models::Reach; + +/// A reach as `[changes] reach` and `--reach` spell it. +pub const fn reach_value(reach: &Reach) -> &'static str { + match reach { + Reach::Job(_) => REACH_JOB, + Reach::All => REACH_ALL, + } +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/relation.rs b/crates/sqlbuild-python-planner/src/consumption/main/relation.rs new file mode 100644 index 0000000000..d3b1b84cdb --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/relation.rs @@ -0,0 +1,15 @@ +//! The relation a model or seed destination names; a destination without a schema has an empty one. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::native_host::main::optional_text::optional_text; +use sqlbuild_state::consumption::models::RelationKey; + +/// The relation a model or seed destination names; a destination without a schema has an empty one. +pub fn relation(destination: &Bound<'_, PyAny>) -> PyResult { + Ok(RelationKey { + database: optional_text(&native_field(destination, "database")?)?, + schema: optional_text(&native_field(destination, "schema")?)?.unwrap_or_default(), + name: native_field(destination, "name")?.extract()?, + }) +} diff --git a/crates/sqlbuild-python-planner/src/consumption/main/run_reason_text.rs b/crates/sqlbuild-python-planner/src/consumption/main/run_reason_text.rs new file mode 100644 index 0000000000..50d60da689 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/main/run_reason_text.rs @@ -0,0 +1,27 @@ +//! Why `--changed` runs a node, as one line for verbose output. + +use sqlbuild_state::consumption::models::{ForcedRun, RunReason}; + +/// Why `--changed` runs a node, as one line for verbose output. +pub fn run_reason_text(reason: &RunReason) -> String { + match reason { + RunReason::Forced(ForcedRun::PlannerChange(kind)) => format!("planner change ({kind})"), + RunReason::Forced(ForcedRun::FullRefresh) => "full refresh".to_owned(), + RunReason::Forced(ForcedRun::RelationMissing) => "relation missing".to_owned(), + RunReason::Forced(ForcedRun::CursorInterval) => "cursor interval requested".to_owned(), + RunReason::Forced(ForcedRun::InputsNotEnumerable) => { + "inputs cannot be enumerated".to_owned() + } + RunReason::NoRecord => "no completed record".to_owned(), + RunReason::Unfinished => "last execution did not finish".to_owned(), + RunReason::AmbiguousOrder => "record order is ambiguous".to_owned(), + RunReason::DefinitionChanged => "definition changed".to_owned(), + RunReason::InputsChanged => "inputs changed".to_owned(), + RunReason::InputChanged(input) => format!("input {input} changed"), + RunReason::UpstreamDefinitionChanged(input) => { + format!("upstream {input} was rebuilt with a definition change") + } + RunReason::InputUnknown(input) => format!("input {input} has no known version"), + RunReason::UpstreamRuns(key) => format!("upstream {} {} runs", key.kind, key.name), + } +} diff --git a/crates/sqlbuild-python-planner/src/consumption/mod.rs b/crates/sqlbuild-python-planner/src/consumption/mod.rs new file mode 100644 index 0000000000..a250790faf --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/mod.rs @@ -0,0 +1,5 @@ +//! What a build consumed: the `--changed` facts and policy, the consumption recorder and relation reads. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-planner/src/consumption/models.rs b/crates/sqlbuild-python-planner/src/consumption/models.rs new file mode 100644 index 0000000000..dbf9c58baf --- /dev/null +++ b/crates/sqlbuild-python-planner/src/consumption/models.rs @@ -0,0 +1,220 @@ +//! Public records of consumption. + +use crate::consumption::_helpers::facts::MODEL; +use crate::consumption::_helpers::facts::SEED; +use crate::consumption::_helpers::facts::enum_text; +use crate::consumption::_helpers::recorder::ERROR_OUTCOME; +use crate::consumption::_helpers::recorder::FLUSH_EXECUTIONS; +use crate::consumption::_helpers::recorder::FLUSH_INTERVAL; +use crate::consumption::_helpers::recorder::Recording; +use crate::consumption::_helpers::recorder::SUCCESS; +use crate::consumption::_helpers::recorder::kind_of; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::native_host::models::ConsumptionConnection; +use sqlbuild_python_front::native_host::models::Location; +use sqlbuild_python_front::native_host::models::NodeCompleteHook; +use sqlbuild_state::consumption::main::consumed_inputs::consumed_inputs; +use sqlbuild_state::consumption::main::succeeded_appends::succeeded_appends; +use sqlbuild_state::consumption::models::ChangedPolicy; +use sqlbuild_state::consumption::models::ConsumptionAppend; +use sqlbuild_state::consumption::models::ConsumptionHistory; +use sqlbuild_state::consumption::models::HeldBack; +use sqlbuild_state::consumption::models::NodeFacts; +use sqlbuild_state::consumption::models::NodeKey; +use sqlbuild_state::consumption::models::RunReason; +use std::collections::BTreeSet; +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::Instant; + +/// Appends `started` rows before a build and `succeeded` rows as its models and seeds complete. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct ConsumptionRecorder { + pub(crate) connection: ConsumptionConnection, + pub(crate) run_id: String, + pub(crate) nodes: HashMap, + pub(crate) history: ConsumptionHistory, + pub(crate) executions: HashMap, + pub(crate) failed_start: HashMap, + pub(crate) base_on_node_complete: NodeCompleteHook, + pub(crate) recording: Mutex, +} + +/// What `--changed` decided, for the build's report. +#[derive(Debug)] +pub struct ChangedDecision { + pub policy: ChangedPolicy, + pub run: usize, + /// Unchanged nodes, which are up to date. + pub skipped: Vec, + /// Nodes downstream of a change that wait for a later run. + pub held_back: Vec<(NodeKey, HeldBack)>, + pub reasons: Vec<(NodeKey, RunReason)>, +} + +impl ChangedDecision { + /// Every selected node `--changed` leaves unbuilt: unchanged and held back. + pub fn not_run(&self) -> impl Iterator { + self.skipped + .iter() + .chain(self.held_back.iter().map(|(key, _)| key)) + } +} + +impl ConsumptionRecorder { + pub(crate) fn lock(&self) -> PyResult> { + self.recording + .lock() + .map_err(|_| PyRuntimeError::new_err("consumption recorder state is poisoned")) + } + + /// The succeeded rows of one execution that wrote its relation, with their schema. + pub(crate) fn succeeded(&self, key: &NodeKey) -> Option<(Location, Vec)> { + let (Some(node), Some(execution_id)) = (self.nodes.get(key), self.executions.get(key)) + else { + return None; + }; + let inputs = consumed_inputs(node, (&self.nodes, &self.history), &self.executions); + Some(( + Location::of(&node.relation), + succeeded_appends(node, (execution_id, &self.run_id), &inputs), + )) + } + + /// The recording with `key`'s succeeded rows queued. + pub(crate) fn queued(&self, mut recording: Recording, key: &NodeKey) -> Recording { + if let Some((location, rows)) = self.succeeded(key) { + recording.pending.entry(location).or_default().extend(rows); + recording.pending_executions += 1; + } + recording + } + + /// The recording after appending every pending row; a failure is reported, and its executions run again. + pub(crate) fn flushed(&self, py: Python<'_>, mut recording: Recording) -> Recording { + let pending = std::mem::take(&mut recording.pending); + let wrote: bool = !pending.is_empty(); + for (location, rows) in pending { + let created: bool = !recording.tables.insert(location.clone()); + let written: PyResult<()> = (if created { + Ok(()) + } else { + self.connection.ensure_table(py, &location) + }) + .and_then(|()| self.connection.append(py, &location, &rows)); + if let Err(error) = written { + recording.warnings.push(format!( + "could not record consumed inputs in schema {}: {error}; those models and seeds run again under --changed", + location.schema + )); + } + } + if wrote && let Err(error) = self.connection.close(py) { + recording.warnings.push(format!( + "could not close the consumed-input connection: {error}" + )); + } + recording.pending_executions = 0; + recording.last_flush = Instant::now(); + recording + } + + /// Queue the seeds whose error audits passed and flush everything. + pub fn finish(&self, py: Python<'_>, build_result: &Bound<'_, PyAny>) -> PyResult> { + let mut failed_audits: BTreeSet = BTreeSet::new(); + for audit in native_field(build_result, "source_audit_results")?.try_iter()? { + let audit: Bound<'_, PyAny> = audit?; + let target: Bound<'_, PyAny> = native_field(&audit, "attached_target_name")?; + if !target.is_none() + && enum_text(&native_field(&audit, "outcome")?)?.eq_ignore_ascii_case(ERROR_OUTCOME) + { + failed_audits.insert(target.extract()?); + } + } + let mut recording = self.lock()?; + for seed in std::mem::take(&mut recording.pending_seeds) { + if !failed_audits.contains(&seed.name) { + *recording = self.queued(std::mem::take(&mut *recording), &seed); + } + } + *recording = self.flushed(py, std::mem::take(&mut *recording)); + Ok(std::mem::take(&mut recording.warnings)) + } +} + +#[pymethods] +impl ConsumptionRecorder { + /// Forward a finished node, then queue a successful model, or a seed until its audits are known. + pub(crate) fn on_node_complete( + &self, + py: Python<'_>, + node_result: Bound<'_, PyAny>, + ) -> PyResult<()> { + self.base_on_node_complete.forward(&node_result)?; + let Some((kind, name)) = kind_of(&node_result)? else { + return Ok(()); + }; + if !enum_text(&native_field(&node_result, "status")?)?.eq_ignore_ascii_case(SUCCESS) { + return Ok(()); + } + let key = NodeKey { + kind: kind.to_owned(), + name, + }; + let mut recording = self.lock()?; + if kind == SEED { + recording.pending_seeds.push(key); + return Ok(()); + } + *recording = self.queued(std::mem::take(&mut *recording), &key); + if recording.pending_executions >= FLUSH_EXECUTIONS + || recording.last_flush.elapsed() >= FLUSH_INTERVAL + { + *recording = self.flushed(py, std::mem::take(&mut *recording)); + } + Ok(()) + } + + /// Fail a model or seed whose `started` row could not be appended, before it writes. + #[pyo3(signature = (*, entry, connection))] + pub(crate) fn before_node_write( + &self, + entry: Bound<'_, PyAny>, + connection: Bound<'_, PyAny>, + ) -> PyResult<()> { + let _ = connection; + let name: String = native_field(&entry, "name")?.extract()?; + let kind: &str = if entry.hasattr("csv_settings")? { + SEED + } else { + MODEL + }; + let key = NodeKey { + kind: kind.to_owned(), + name, + }; + match self.failed_start.get(&key) { + Some(error) => Err(PyRuntimeError::new_err(format!( + "not written: SQLBuild could not record that this {kind} started ({error}), so a crash could not be detected later" + ))), + None => Ok(()), + } + } +} + +/// Everything the build needs from the recorder: the recorder itself and what `--changed` skipped. +#[derive(Debug)] +pub struct Prepared<'py> { + pub recorder: Bound<'py, ConsumptionRecorder>, + pub decision: Option, + pub warnings: Vec, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/exception_types.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/exception_types.rs similarity index 53% rename from crates/sqlbuild-python/src/bindings/_helpers/freshness/exception_types.rs rename to crates/sqlbuild-python-planner/src/freshness/_helpers/exception_types.rs index 8d6c959c03..0ddcd1b496 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/exception_types.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/exception_types.rs @@ -2,7 +2,6 @@ use pyo3::create_exception; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; create_exception!( sqlbuild._native, @@ -16,15 +15,3 @@ create_exception!( PyValueError, "Raised when source freshness cannot be observed safely." ); - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - let py = module.py(); - module.add( - "SourceFreshnessInputError", - py.get_type::(), - )?; - module.add( - "SourceFreshnessObservationError", - py.get_type::(), - ) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs rename to crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs index a7712a43d3..37f08a5d38 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs @@ -1,21 +1,20 @@ //! Observe the current freshness of selected sources and compare it with previous state. +use crate::freshness::models::PreviousState; use std::collections::{HashMap, HashSet}; use pyo3::exceptions::{PyException, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::freshness_text::main::freshness_json::freshness_json; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::freshness::observation::{ - Observer, observation_record, observation_source, observe_adapter_batch, observe_configured, - strategy, +use crate::freshness::_helpers::observation::{ + observation_record, observation_source, observe_adapter_batch, observe_configured, strategy, }; +use crate::freshness::models::Observer; use sqlbuild_core::freshness_text::main::freshness_status::freshness_status; use sqlbuild_core::freshness_text::models::{ FreshnessSourceResult, FreshnessStatus, StateComparison, @@ -34,11 +33,6 @@ const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavai const MISSING_STATE_MESSAGE: &str = "previous source freshness state missing"; const NOT_RETURNED_MESSAGE: &str = "freshness metadata was not returned"; -/// Previous state records by identity; absent means no comparison. -pub(crate) struct PreviousState<'py> { - pub(crate) by_identity: Option>, -} - fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { if value.is_none() { return Ok(None); @@ -300,7 +294,7 @@ pub(crate) fn observe_freshness( /// Observe source freshness and return the `sqb freshness --json` payload. #[pyfunction] #[pyo3(signature = (**options))] -fn observe_source_freshness_json( +pub(crate) fn observe_source_freshness_json( py: Python<'_>, options: Option>, ) -> PyResult { @@ -340,8 +334,3 @@ fn observe_source_freshness_json( )?; freshness_json(&results).map_err(|error| PyValueError::new_err(format!("{error:?}"))) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(observe_source_freshness_json, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-planner/src/freshness/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/mod.rs new file mode 100644 index 0000000000..70eb32c83b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod exception_types; +pub(crate) mod freshness_observe; +pub(crate) mod observation; +pub(crate) mod planning; +pub(crate) mod publication; +pub(crate) mod state_io; +pub(crate) mod support; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs rename to crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs index 873f93551f..cc734a6bcf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs @@ -1,21 +1,20 @@ //! Observe configured and adapter-metadata source freshness through adapter callbacks. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDateTime, PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::freshness::main::presumed_utc_datetime::presumed_utc_datetime; +use crate::freshness::models::Observer; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use sqlbuild_state::freshness::main::single_value_problem::single_value_problem; use sqlbuild_state::freshness::models::QueryShape; -use crate::bindings::_helpers::freshness::support::{ - OBSERVATION_ERROR, exception_text, freshness_error, presumed_utc_datetime, presumed_utc_value, - text, +use crate::freshness::_helpers::support::{ + OBSERVATION_ERROR, exception_text, freshness_error, presumed_utc_value, text, }; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::constants::DATA_VERSION_FIELD; -use sqlbuild_python_base::records::constants::OBSERVED_AT_FIELD; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -24,9 +23,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_front::state_rows::main::event_rows::in_utc::in_utc; use sqlbuild_python_front::state_rows::main::event_rows::normalize::normalize; -use sqlbuild_python_front::state_rows::main::event_rows::offset_aware::offset_aware; use sqlbuild_python_front::state_rows::main::event_rows::version_hash::version_hash; use sqlbuild_python_project::references::main::render_source_read::render_source_read; @@ -36,14 +33,7 @@ const DEFAULT_STRATEGY: &str = "sql"; const METADATA_OPERATION: &str = "source_freshness_metadata_observation"; const QUERY_OPERATION: &str = "source_freshness_query_observation"; -/// The adapter, connection and observation time every source is observed with. -pub(crate) struct Observer<'py> { - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) connection: Bound<'py, PyAny>, - pub(crate) observed_at: Bound<'py, PyAny>, -} - -fn observation_error(py: Python<'_>, message: &str) -> pyo3::PyErr { +pub(crate) fn observation_error(py: Python<'_>, message: &str) -> pyo3::PyErr { freshness_error(py, (OBSERVATION_ERROR, message), None) } @@ -123,7 +113,7 @@ fn lifecycle<'py>( call_with(&host_entry(py, ExecutorHost::OperationLifecycle)?, &options) } -fn observation<'py>( +pub(crate) fn observation<'py>( source_name: &Bound<'py, PyAny>, (strategy, value_kind): (Bound<'py, PyAny>, Bound<'py, PyAny>), (data_version, observed_at): (Bound<'py, PyAny>, Bound<'py, PyAny>), @@ -559,7 +549,7 @@ pub(crate) fn observation_record<'py>( /// Observe one source freshness config and return a comparable data version. #[pyfunction] #[pyo3(signature = (*, adapter, connection, source, observed_at))] -fn observe_configured_source_freshness<'py>( +pub(crate) fn observe_configured_source_freshness<'py>( adapter: Bound<'py, PyAny>, connection: Bound<'py, PyAny>, source: &Bound<'py, PyAny>, @@ -576,7 +566,7 @@ fn observe_configured_source_freshness<'py>( /// Observe adapter metadata freshness per source, batching the warehouse lookups. #[pyfunction] #[pyo3(signature = (*, adapter, connection, sources, observed_at))] -fn observe_adapter_sources_freshness<'py>( +pub(crate) fn observe_adapter_sources_freshness<'py>( adapter: Bound<'py, PyAny>, connection: Bound<'py, PyAny>, sources: &Bound<'py, PyAny>, @@ -590,43 +580,3 @@ fn observe_adapter_sources_freshness<'py>( let sources: Vec> = sources.try_iter()?.collect::>()?; observe_adapter_batch(&observer, &sources) } - -/// An aware freshness `datetime` in UTC; a naive value is an observation error. -pub(crate) fn require_aware_utc_datetime<'py>( - value: &Bound<'py, PyAny>, - field_name: &str, -) -> PyResult> { - if !offset_aware(value)? { - return Err(observation_error( - value.py(), - &format!("source freshness {field_name} must be timezone-aware"), - )); - } - in_utc(value) -} - -/// Hold an observation's `observed_at` and datetime `data_version` as aware UTC values. -pub(crate) fn normalize_observation( - py: Python<'_>, - mut values: Vec>, -) -> PyResult>> { - let observed_at: Bound<'_, PyAny> = - require_aware_utc_datetime(values[OBSERVED_AT_FIELD].bind(py), "observed_at")?; - values[OBSERVED_AT_FIELD] = observed_at.unbind(); - let data_version: &Bound<'_, PyAny> = values[DATA_VERSION_FIELD].bind(py); - if data_version.is_instance_of::() { - let data_version: Bound<'_, PyAny> = - require_aware_utc_datetime(data_version, "data_version")?; - values[DATA_VERSION_FIELD] = data_version.unbind(); - } - Ok(values) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - observe_configured_source_freshness, - module - )?)?; - module.add_function(wrap_pyfunction!(observe_adapter_sources_freshness, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs rename to crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs index 14b5a97b82..4829e50f12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs @@ -1,25 +1,26 @@ //! Planning-time source freshness: observe, compare with latest state and map downstream impact. +use crate::freshness::constants::PREVIEW_MESSAGE; use std::collections::{HashMap, HashSet}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::planner_identity::constants::{MODEL_RESOURCE, SOURCE_RESOURCE}; use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_state::freshness::main::configuration_error::configuration_error; -use crate::bindings::_helpers::freshness::exception_types::SourceFreshnessObservationError; -use crate::bindings::_helpers::freshness::observation::{ - Observer, observation_record, observation_source, observe_adapter_batch, observe_configured, - strategy, unknown, +use crate::freshness::_helpers::exception_types::SourceFreshnessObservationError; +use crate::freshness::_helpers::observation::{ + observation_record, observation_source, observe_adapter_batch, observe_configured, strategy, + unknown, }; -use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; -use crate::bindings::_helpers::freshness::support::text; +use crate::freshness::_helpers::state_io::read_latest; +use crate::freshness::_helpers::support::text; +use crate::freshness::models::Observer; +use crate::freshness::models::StateRead; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; @@ -35,8 +36,6 @@ use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; const ADAPTER_STRATEGY: &str = "adapter"; const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavailable"; -/// Why a plan preview leaves a source's freshness unknown; its plan lists freshness as unknown. -pub(crate) const PREVIEW_MESSAGE: &str = "a plan preview does not observe source freshness"; /// The message of an adapter, freshness observation or preview access error, if it is one. fn handled_message<'py>(py: Python<'py>, error: &PyErr) -> PyResult>> { @@ -391,15 +390,3 @@ pub(crate) fn build_direct_source_freshness_propagation_result<'py>( ], ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - build_direct_source_freshness_planning_result, - module - )?)?; - module.add_function(wrap_pyfunction!( - build_direct_source_freshness_propagation_result, - module - )?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/publication.rs similarity index 79% rename from crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs rename to crates/sqlbuild-python-planner/src/freshness/_helpers/publication.rs index c7a628f3a1..b43220bb06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/publication.rs @@ -1,13 +1,12 @@ //! Append observed source freshness to state once the models it affected have built. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::freshness::main::append_freshness_records::append_freshness_records; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::cost_telemetry::context::CostContext; -use crate::bindings::_helpers::freshness::state_io::write_source_freshness_records; -use sqlbuild_python_base::boundary::main::call_with::call_with; +use crate::freshness::_helpers::state_io::write_source_freshness_records; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; @@ -18,9 +17,10 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cost_context::models::CostContext; /// Observed records grouped by `(database, schema)` in first-seen order. -type RecordsByLocation<'py> = Vec<( +pub(crate) type RecordsByLocation<'py> = Vec<( (Bound<'py, PyAny>, Bound<'py, PyAny>), Vec>, )>; @@ -103,7 +103,7 @@ fn contains_location( } /// The observed records to publish, keyed by the schema of each affected model. -fn eligible_records<'py>( +pub(crate) fn eligible_records<'py>( plan: &Bound<'py, PyAny>, successful_models: &[String], run_id: &Bound<'py, PyAny>, @@ -188,7 +188,7 @@ fn renderers<'py>(adapter: &Bound<'py, PyAny>) -> PyResult> { } /// Write each location's records, one cost scope per source. -fn write_records<'py>( +pub(crate) fn write_records<'py>( adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, grouped: &RecordsByLocation<'py>, @@ -250,58 +250,10 @@ fn write_records<'py>( Ok(()) } -/// Append observed source freshness only after the affected selected models succeed. -pub(crate) fn append_freshness_records<'py>( - (plan, successful_models): (&Bound<'py, PyAny>, &[String]), - adapter: &Bound<'py, PyAny>, - connection_config: &Bound<'py, PyAny>, - run_id: &Bound<'py, PyAny>, -) -> PyResult<()> { - let py: Python<'py> = plan.py(); - let grouped: RecordsByLocation<'py> = eligible_records(plan, successful_models, run_id)?; - let record_count: usize = grouped.iter().map(|(_, records)| records.len()).sum(); - if record_count == 0 { - return Ok(()); - } - let run_scope: Bound<'py, PyAny> = host_entry(py, ExecutorHost::RunScope)?.call1((run_id,))?; - within(&run_scope, |_| { - let metadata: Bound<'py, PyDict> = PyDict::new(py); - metadata.set_item("item_count", record_count)?; - let attributes: Bound<'py, PyAny> = call_with( - &host_entry(py, ExecutorHost::OperationAttributes)?, - &[ - ("phase", PyString::new(py, "publish").into_any()), - ("target_kind", PyString::new(py, "state_batch").into_any()), - ], - )?; - let lifecycle: Bound<'py, PyAny> = call_with( - &host_entry(py, ExecutorHost::OperationLifecycle)?, - &[ - ("operation_kind", PyString::new(py, "freshness").into_any()), - ( - "operation_name", - PyString::new(py, "source_freshness_publication").into_any(), - ), - ("metadata", metadata.into_any()), - ("attributes", attributes), - ], - )?; - within(&lifecycle, |_| { - let connection: Bound<'py, PyAny> = - adapter.call_method1("connect", (connection_config,))?; - let written: PyResult<()> = write_records(adapter, &connection, &grouped); - let closed: PyResult> = adapter.call_method1("close", (&connection,)); - written?; - closed?; - Ok(()) - }) - }) -} - /// Append observed source freshness only after the affected selected models succeed. #[pyfunction] #[pyo3(signature = (**options))] -fn append_eligible_direct_source_freshness_records<'py>( +pub(crate) fn append_eligible_direct_source_freshness_records<'py>( py: Python<'py>, options: Option>, ) -> PyResult<()> { @@ -324,10 +276,3 @@ fn append_eligible_direct_source_freshness_records<'py>( &arguments.value("run_id")?, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - append_eligible_direct_source_freshness_records, - module - )?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs similarity index 79% rename from crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs rename to crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs index 317e8a32e4..6bcbeed0ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs @@ -1,8 +1,10 @@ //! Source freshness state table DDL, latest-row reads and appends through adapter callbacks. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::freshness::main::row_record::row_record; +use crate::freshness::models::StateRead; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; use sqlbuild_state::freshness::constants::{ REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_COLUMN_TYPES, SOURCE_FRESHNESS_EVENT_ID, @@ -11,19 +13,14 @@ use sqlbuild_state::freshness::constants::{ use sqlbuild_state::freshness::main::read_latest_sql::read_latest_sql; use sqlbuild_state::freshness::main::relevant_sources_sql::relevant_sources_sql; -use crate::bindings::_helpers::freshness::support::{ - INPUT_ERROR, exception_text, freshness_error, optional_str, presumed_utc_datetime, text, -}; +use crate::freshness::_helpers::support::{INPUT_ERROR, exception_text, freshness_error, text}; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::main::ensure_event_id_column::ensure_event_id_column; -use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; use sqlbuild_python_front::native_host::main::state_table_name::state_table_name; -use sqlbuild_python_front::state_rows::main::event_rows::date_time_typed::date_time_typed; use sqlbuild_python_front::state_rows::main::tables::state_table_create_sql::state_table_create_sql; const MISSING_SCHEMA_MESSAGE: &str = "source freshness table requires a target schema"; @@ -47,7 +44,7 @@ pub(crate) fn qualified_table_name( /// SQL selecting the latest source freshness row per identity. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name))] -fn build_source_freshness_read_latest_sql( +pub(crate) fn build_source_freshness_read_latest_sql( database: &Bound<'_, PyAny>, schema: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, @@ -61,7 +58,7 @@ fn build_source_freshness_read_latest_sql( /// Create-if-missing DDL for the source freshness state table. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name, render_framework_type))] -fn build_source_freshness_create_table_sql<'py>( +pub(crate) fn build_source_freshness_create_table_sql<'py>( database: &Bound<'py, PyAny>, schema: &Bound<'py, PyAny>, render_qualified_name: &Bound<'py, PyAny>, @@ -105,14 +102,6 @@ fn create_table_sql<'py>( Ok(text(py, &sql)) } -/// The callbacks and location one latest-state read uses. -pub(crate) struct StateRead<'py> { - pub(crate) connection: Bound<'py, PyAny>, - pub(crate) execute: Bound<'py, PyAny>, - pub(crate) render_qualified_name: Bound<'py, PyAny>, - pub(crate) render_read_latest_sql: Bound<'py, PyAny>, -} - fn empty_set<'py>(schema: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = schema.py(); new_record_with( @@ -187,43 +176,10 @@ freshness table to regenerate source freshness state." ) } -pub(crate) fn row_record<'py>(row: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = row.py(); - let raw_observed_at: Bound<'py, PyAny> = row.get_item(9)?; - let observed_at: Bound<'py, PyAny> = if date_time_typed(&raw_observed_at)? { - raw_observed_at - } else { - imported(py, DATETIME_MODULE, "datetime")? - .call_method1("fromisoformat", (raw_observed_at.str()?,))? - }; - let observed_at: Bound<'py, PyAny> = presumed_utc_datetime(&observed_at)?; - let field = |index: usize| -> PyResult> { - Ok(row.get_item(index)?.str()?.into_any()) - }; - let optional = - |index: usize| -> PyResult> { optional_str(&row.get_item(index)?) }; - new_record_with( - py, - RecordId::SourceFreshnessRecord, - &[ - ("source_name", field(0)?), - ("target_database", optional(1)?), - ("target_schema", optional(2)?), - ("target_name", optional(3)?), - ("run_id", field(4)?), - ("strategy", field(5)?), - ("value_kind", field(6)?), - ("data_version", optional(7)?), - ("data_version_hash", field(8)?), - ("observed_at", observed_at), - ], - ) -} - /// Read latest source freshness rows, trusting the caller-resolved table existence. #[pyfunction] #[pyo3(signature = (**options))] -fn read_latest_source_freshness<'py>( +pub(crate) fn read_latest_source_freshness<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -341,17 +297,3 @@ pub(crate) fn write_source_freshness_records<'py>( )?; execute_sql((execute, connection), insert_sql) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - build_source_freshness_read_latest_sql, - module - )?)?; - module.add_function(wrap_pyfunction!( - build_source_freshness_create_table_sql, - module - )?)?; - module.add_function(wrap_pyfunction!(read_latest_source_freshness, module)?)?; - module.add_function(wrap_pyfunction!(write_source_freshness_records, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs b/crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs similarity index 69% rename from crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs rename to crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs index 93d3fb0850..f4b87f48bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs +++ b/crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs @@ -1,17 +1,14 @@ //! Python objects, exceptions and datetimes the source freshness glue shares. +use crate::freshness::main::presumed_utc_datetime::presumed_utc_datetime; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyString}; +use pyo3::types::PyString; -use crate::bindings::_helpers::freshness::exception_types::{ +use crate::freshness::_helpers::exception_types::{ SourceFreshnessInputError, SourceFreshnessObservationError, }; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; use sqlbuild_python_front::state_rows::main::event_rows::date_time_typed::date_time_typed; -use sqlbuild_python_front::state_rows::main::event_rows::in_utc::in_utc; -use sqlbuild_python_front::state_rows::main::event_rows::offset_aware::offset_aware; pub(crate) const OBSERVATION_ERROR: fn(String) -> PyErr = SourceFreshnessObservationError::new_err; pub(crate) const INPUT_ERROR: fn(String) -> PyErr = SourceFreshnessInputError::new_err; @@ -45,16 +42,6 @@ pub(crate) fn presumed_utc_value<'py>(value: &Bound<'py, PyAny>) -> PyResult(value: &Bound<'py, PyAny>) -> PyResult> { - if offset_aware(value)? { - return in_utc(value); - } - let changes: Bound<'py, PyDict> = PyDict::new(value.py()); - changes.set_item("tzinfo", imported(value.py(), DATETIME_MODULE, "UTC")?)?; - value.call_method("replace", (), Some(&changes)) -} - pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() } diff --git a/crates/sqlbuild-python-planner/src/freshness/constants.rs b/crates/sqlbuild-python-planner/src/freshness/constants.rs new file mode 100644 index 0000000000..3c1731b37e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/constants.rs @@ -0,0 +1,4 @@ +//! Public constants of freshness. + +/// Why a plan preview leaves a source's freshness unknown; its plan lists freshness as unknown. +pub const PREVIEW_MESSAGE: &str = "a plan preview does not observe source freshness"; diff --git a/crates/sqlbuild-python-planner/src/freshness/main/append_freshness_records.rs b/crates/sqlbuild-python-planner/src/freshness/main/append_freshness_records.rs new file mode 100644 index 0000000000..8f1323b136 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/append_freshness_records.rs @@ -0,0 +1,59 @@ +//! Append observed source freshness only after the affected selected models succeed. + +use crate::freshness::_helpers::publication::RecordsByLocation; +use crate::freshness::_helpers::publication::eligible_records; +use crate::freshness::_helpers::publication::write_records; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; + +/// Append observed source freshness only after the affected selected models succeed. +pub fn append_freshness_records<'py>( + (plan, successful_models): (&Bound<'py, PyAny>, &[String]), + adapter: &Bound<'py, PyAny>, + connection_config: &Bound<'py, PyAny>, + run_id: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py: Python<'py> = plan.py(); + let grouped: RecordsByLocation<'py> = eligible_records(plan, successful_models, run_id)?; + let record_count: usize = grouped.iter().map(|(_, records)| records.len()).sum(); + if record_count == 0 { + return Ok(()); + } + let run_scope: Bound<'py, PyAny> = host_entry(py, ExecutorHost::RunScope)?.call1((run_id,))?; + within(&run_scope, |_| { + let metadata: Bound<'py, PyDict> = PyDict::new(py); + metadata.set_item("item_count", record_count)?; + let attributes: Bound<'py, PyAny> = call_with( + &host_entry(py, ExecutorHost::OperationAttributes)?, + &[ + ("phase", PyString::new(py, "publish").into_any()), + ("target_kind", PyString::new(py, "state_batch").into_any()), + ], + )?; + let lifecycle: Bound<'py, PyAny> = call_with( + &host_entry(py, ExecutorHost::OperationLifecycle)?, + &[ + ("operation_kind", PyString::new(py, "freshness").into_any()), + ( + "operation_name", + PyString::new(py, "source_freshness_publication").into_any(), + ), + ("metadata", metadata.into_any()), + ("attributes", attributes), + ], + )?; + within(&lifecycle, |_| { + let connection: Bound<'py, PyAny> = + adapter.call_method1("connect", (connection_config,))?; + let written: PyResult<()> = write_records(adapter, &connection, &grouped); + let closed: PyResult> = adapter.call_method1("close", (&connection,)); + written?; + closed?; + Ok(()) + }) + }) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/build_direct_source_freshness_planning_result_function.rs b/crates/sqlbuild-python-planner/src/freshness/main/build_direct_source_freshness_planning_result_function.rs new file mode 100644 index 0000000000..0a2539f738 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/build_direct_source_freshness_planning_result_function.rs @@ -0,0 +1,10 @@ +//! `build_direct_source_freshness_planning_result` as a Python function. + +use crate::freshness::_helpers::planning::build_direct_source_freshness_planning_result; + +/// `build_direct_source_freshness_planning_result` as a Python function. +pub fn build_direct_source_freshness_planning_result_function( + py: pyo3::Python<'_>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_direct_source_freshness_planning_result, py) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/build_direct_source_freshness_propagation_result_function.rs b/crates/sqlbuild-python-planner/src/freshness/main/build_direct_source_freshness_propagation_result_function.rs new file mode 100644 index 0000000000..7ba669dcef --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/build_direct_source_freshness_propagation_result_function.rs @@ -0,0 +1,10 @@ +//! `build_direct_source_freshness_propagation_result` as a Python function. + +use crate::freshness::_helpers::planning::build_direct_source_freshness_propagation_result; + +/// `build_direct_source_freshness_propagation_result` as a Python function. +pub fn build_direct_source_freshness_propagation_result_function( + py: pyo3::Python<'_>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_direct_source_freshness_propagation_result, py) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/mod.rs b/crates/sqlbuild-python-planner/src/freshness/main/mod.rs new file mode 100644 index 0000000000..7fba126b1e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/mod.rs @@ -0,0 +1,15 @@ +pub mod append_freshness_records; +pub mod build_direct_source_freshness_planning_result_function; +pub mod build_direct_source_freshness_propagation_result_function; +pub mod normalize_observation; +pub mod observe_freshness; +pub mod presumed_utc_datetime; +pub mod read_latest; +pub mod register_exception_types; +pub mod register_freshness_observe; +pub mod register_observation; +pub mod register_planning; +pub mod register_publication; +pub mod register_state_io; +pub mod require_aware_utc_datetime; +pub mod row_record; diff --git a/crates/sqlbuild-python-planner/src/freshness/main/normalize_observation.rs b/crates/sqlbuild-python-planner/src/freshness/main/normalize_observation.rs new file mode 100644 index 0000000000..15d199d999 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/normalize_observation.rs @@ -0,0 +1,24 @@ +//! Hold an observation's `observed_at` and datetime `data_version` as aware UTC values. + +use crate::freshness::main::require_aware_utc_datetime::require_aware_utc_datetime; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDateTime; +use sqlbuild_python_base::records::constants::DATA_VERSION_FIELD; +use sqlbuild_python_base::records::constants::OBSERVED_AT_FIELD; + +/// Hold an observation's `observed_at` and datetime `data_version` as aware UTC values. +pub fn normalize_observation( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + let observed_at: Bound<'_, PyAny> = + require_aware_utc_datetime(values[OBSERVED_AT_FIELD].bind(py), "observed_at")?; + values[OBSERVED_AT_FIELD] = observed_at.unbind(); + let data_version: &Bound<'_, PyAny> = values[DATA_VERSION_FIELD].bind(py); + if data_version.is_instance_of::() { + let data_version: Bound<'_, PyAny> = + require_aware_utc_datetime(data_version, "data_version")?; + values[DATA_VERSION_FIELD] = data_version.unbind(); + } + Ok(values) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/observe_freshness.rs b/crates/sqlbuild-python-planner/src/freshness/main/observe_freshness.rs new file mode 100644 index 0000000000..1732f96b79 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/observe_freshness.rs @@ -0,0 +1,17 @@ +//! Observe the selected sources inside the run's observability scope. + +use crate::freshness::_helpers::freshness_observe; +use crate::freshness::models::Observer; +use crate::freshness::models::PreviousState; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_core::freshness_text::models::FreshnessSourceResult; + +/// Observe the selected sources inside the run's observability scope. +#[inline] +pub fn observe_freshness( + observer: &Observer<'_>, + (sources, select, exclude): (&Bound<'_, PyAny>, &[String], &[String]), + (run_id, state): (Option, &PreviousState<'_>), +) -> PyResult> { + freshness_observe::observe_freshness(observer, (sources, select, exclude), (run_id, state)) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/presumed_utc_datetime.rs b/crates/sqlbuild-python-planner/src/freshness/main/presumed_utc_datetime.rs new file mode 100644 index 0000000000..98f2ecd68e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/presumed_utc_datetime.rs @@ -0,0 +1,18 @@ +//! A `datetime` in UTC, a naive or offset-less value read as UTC. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; +use sqlbuild_python_front::state_rows::main::event_rows::in_utc::in_utc; +use sqlbuild_python_front::state_rows::main::event_rows::offset_aware::offset_aware; + +/// A `datetime` in UTC, a naive or offset-less value read as UTC. +pub fn presumed_utc_datetime<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if offset_aware(value)? { + return in_utc(value); + } + let changes: Bound<'py, PyDict> = PyDict::new(value.py()); + changes.set_item("tzinfo", imported(value.py(), DATETIME_MODULE, "UTC")?)?; + value.call_method("replace", (), Some(&changes)) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/read_latest.rs b/crates/sqlbuild-python-planner/src/freshness/main/read_latest.rs new file mode 100644 index 0000000000..c1f899cf88 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/read_latest.rs @@ -0,0 +1,16 @@ +//! Latest records of one schema, trusting the caller-resolved table existence. + +use crate::freshness::_helpers::state_io; +use crate::freshness::models::StateRead; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Latest records of one schema, trusting the caller-resolved table existence. +#[inline] +pub fn read_latest<'py>( + read: &StateRead<'py>, + (database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + table_exists: bool, + source_names: Option<&[String]>, +) -> PyResult> { + state_io::read_latest(read, (database, schema), table_exists, source_names) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_exception_types.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_exception_types.rs new file mode 100644 index 0000000000..f3aa7b9dce --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_exception_types.rs @@ -0,0 +1,18 @@ +//! Register the exception types functions and classes. + +use crate::freshness::_helpers::exception_types::SourceFreshnessInputError; +use crate::freshness::_helpers::exception_types::SourceFreshnessObservationError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the exception types functions and classes. +pub fn register_exception_types(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py = module.py(); + module.add( + "SourceFreshnessInputError", + py.get_type::(), + )?; + module.add( + "SourceFreshnessObservationError", + py.get_type::(), + ) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_freshness_observe.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_freshness_observe.rs new file mode 100644 index 0000000000..fcc9d6671d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_freshness_observe.rs @@ -0,0 +1,11 @@ +//! Register the freshness observe functions and classes. + +use crate::freshness::_helpers::freshness_observe::observe_source_freshness_json; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the freshness observe functions and classes. +pub fn register_freshness_observe(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(observe_source_freshness_json, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_observation.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_observation.rs new file mode 100644 index 0000000000..291cd0eef0 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_observation.rs @@ -0,0 +1,16 @@ +//! Register the observation functions and classes. + +use crate::freshness::_helpers::observation::observe_adapter_sources_freshness; +use crate::freshness::_helpers::observation::observe_configured_source_freshness; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the observation functions and classes. +pub fn register_observation(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + observe_configured_source_freshness, + module + )?)?; + module.add_function(wrap_pyfunction!(observe_adapter_sources_freshness, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_planning.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_planning.rs new file mode 100644 index 0000000000..a1563d226a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_planning.rs @@ -0,0 +1,19 @@ +//! Register the planning functions and classes. + +use crate::freshness::_helpers::planning::build_direct_source_freshness_planning_result; +use crate::freshness::_helpers::planning::build_direct_source_freshness_propagation_result; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the planning functions and classes. +pub fn register_planning(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_direct_source_freshness_planning_result, + module + )?)?; + module.add_function(wrap_pyfunction!( + build_direct_source_freshness_propagation_result, + module + )?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_publication.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_publication.rs new file mode 100644 index 0000000000..ab766f21cf --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_publication.rs @@ -0,0 +1,13 @@ +//! Register the publication functions and classes. + +use crate::freshness::_helpers::publication::append_eligible_direct_source_freshness_records; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the publication functions and classes. +pub fn register_publication(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + append_eligible_direct_source_freshness_records, + module + )?) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs b/crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs new file mode 100644 index 0000000000..b3ac616430 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs @@ -0,0 +1,23 @@ +//! Register the state io functions and classes. + +use crate::freshness::_helpers::state_io::build_source_freshness_create_table_sql; +use crate::freshness::_helpers::state_io::build_source_freshness_read_latest_sql; +use crate::freshness::_helpers::state_io::read_latest_source_freshness; +use crate::freshness::_helpers::state_io::write_source_freshness_records; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the state io functions and classes. +pub fn register_state_io(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_source_freshness_read_latest_sql, + module + )?)?; + module.add_function(wrap_pyfunction!( + build_source_freshness_create_table_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(read_latest_source_freshness, module)?)?; + module.add_function(wrap_pyfunction!(write_source_freshness_records, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/require_aware_utc_datetime.rs b/crates/sqlbuild-python-planner/src/freshness/main/require_aware_utc_datetime.rs new file mode 100644 index 0000000000..376406fbac --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/require_aware_utc_datetime.rs @@ -0,0 +1,20 @@ +//! An aware freshness `datetime` in UTC; a naive value is an observation error. + +use crate::freshness::_helpers::observation::observation_error; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_front::state_rows::main::event_rows::in_utc::in_utc; +use sqlbuild_python_front::state_rows::main::event_rows::offset_aware::offset_aware; + +/// An aware freshness `datetime` in UTC; a naive value is an observation error. +pub fn require_aware_utc_datetime<'py>( + value: &Bound<'py, PyAny>, + field_name: &str, +) -> PyResult> { + if !offset_aware(value)? { + return Err(observation_error( + value.py(), + &format!("source freshness {field_name} must be timezone-aware"), + )); + } + in_utc(value) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/main/row_record.rs b/crates/sqlbuild-python-planner/src/freshness/main/row_record.rs new file mode 100644 index 0000000000..c96bd0218c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/main/row_record.rs @@ -0,0 +1,44 @@ +//! Row record. + +use crate::freshness::_helpers::support::optional_str; +use crate::freshness::main::presumed_utc_datetime::presumed_utc_datetime; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; +use sqlbuild_python_front::state_rows::main::event_rows::date_time_typed::date_time_typed; + +/// Row record. +pub fn row_record<'py>(row: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = row.py(); + let raw_observed_at: Bound<'py, PyAny> = row.get_item(9)?; + let observed_at: Bound<'py, PyAny> = if date_time_typed(&raw_observed_at)? { + raw_observed_at + } else { + imported(py, DATETIME_MODULE, "datetime")? + .call_method1("fromisoformat", (raw_observed_at.str()?,))? + }; + let observed_at: Bound<'py, PyAny> = presumed_utc_datetime(&observed_at)?; + let field = |index: usize| -> PyResult> { + Ok(row.get_item(index)?.str()?.into_any()) + }; + let optional = + |index: usize| -> PyResult> { optional_str(&row.get_item(index)?) }; + new_record_with( + py, + RecordId::SourceFreshnessRecord, + &[ + ("source_name", field(0)?), + ("target_database", optional(1)?), + ("target_schema", optional(2)?), + ("target_name", optional(3)?), + ("run_id", field(4)?), + ("strategy", field(5)?), + ("value_kind", field(6)?), + ("data_version", optional(7)?), + ("data_version_hash", field(8)?), + ("observed_at", observed_at), + ], + ) +} diff --git a/crates/sqlbuild-python-planner/src/freshness/mod.rs b/crates/sqlbuild-python-planner/src/freshness/mod.rs new file mode 100644 index 0000000000..f3bac990ff --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/mod.rs @@ -0,0 +1,6 @@ +//! Source freshness: observation, planning, publication and state, through adapter callbacks. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-planner/src/freshness/models.rs b/crates/sqlbuild-python-planner/src/freshness/models.rs new file mode 100644 index 0000000000..ba73434ddc --- /dev/null +++ b/crates/sqlbuild-python-planner/src/freshness/models.rs @@ -0,0 +1,27 @@ +//! Public records of freshness. + +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; + +/// Previous state records by identity; absent means no comparison. +#[derive(Debug)] +pub struct PreviousState<'py> { + pub by_identity: Option>, +} + +/// The adapter, connection and observation time every source is observed with. +#[derive(Debug)] +pub struct Observer<'py> { + pub adapter: Bound<'py, PyAny>, + pub connection: Bound<'py, PyAny>, + pub observed_at: Bound<'py, PyAny>, +} + +/// The callbacks and location one latest-state read uses. +#[derive(Debug)] +pub struct StateRead<'py> { + pub connection: Bound<'py, PyAny>, + pub execute: Bound<'py, PyAny>, + pub render_qualified_name: Bound<'py, PyAny>, + pub render_read_latest_sql: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python-planner/src/lib.rs b/crates/sqlbuild-python-planner/src/lib.rs new file mode 100644 index 0000000000..ee655109a6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/lib.rs @@ -0,0 +1,19 @@ +//! The plan layer of the `sqlbuild._native` bindings: execution plans and their entries, planner phases, changes, migrations, resolution, scenarios, strategy and warehouse state, freshness, consumption, microbatches, observations, lifecycle events and plan output. + +#![forbid(unsafe_code)] + +pub mod consumption; +pub mod freshness; +pub mod lifecycle_events; +pub mod microbatches; +pub mod observations; +pub mod plan_output; +pub mod planner_changes; +pub mod planner_entries; +pub mod planner_identity; +pub mod planner_migrations; +pub mod planner_phases; +pub mod planner_resolution; +pub mod planner_scenario; +pub mod planner_strategy; +pub mod planner_warehouse; diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/catalog.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/catalog.rs new file mode 100644 index 0000000000..daa4233c91 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/catalog.rs @@ -0,0 +1,46 @@ +//! The native lifecycle event catalogue: known types, terminality and catalogued values. + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyString; +use sqlbuild_core::lifecycle_events::main::catalog::catalog; +use sqlbuild_core::lifecycle_events::models::EventDefinition; + +/// The definition of `event_type` in the `schema_version` catalogue, if both are catalogued. +pub(crate) fn definition( + schema_version: &Bound<'_, PyAny>, + event_type: &Bound<'_, PyAny>, +) -> PyResult> { + let (Ok(version), Ok(name)) = ( + schema_version.extract::(), + event_type.downcast::(), + ) else { + return Ok(None); + }; + let name: String = name.extract()?; + Ok(catalog(version) + .and_then(|entries| entries.into_iter().find(|entry| entry.event_type == name))) +} + +/// Whether `event_type` is catalogued for `schema_version`. +#[pyfunction] +#[pyo3(signature = (*, schema_version, event_type))] +pub(crate) fn lifecycle_event_catalogued( + schema_version: &Bound<'_, PyAny>, + event_type: &Bound<'_, PyAny>, +) -> PyResult { + Ok(definition(schema_version, event_type)?.is_some()) +} + +/// Whether a catalogued event closes its correlated lifecycle scope; `KeyError` otherwise. +#[pyfunction] +#[pyo3(signature = (*, schema_version, event_type))] +pub(crate) fn lifecycle_event_terminal( + schema_version: &Bound<'_, PyAny>, + event_type: &Bound<'_, PyAny>, +) -> PyResult { + definition(schema_version, event_type)? + .map(|entry| entry.terminal) + .ok_or_else(|| PyKeyError::new_err(event_type.clone().unbind())) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs rename to crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs index 5874ec0270..a7a4410d8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs @@ -1,11 +1,11 @@ //! Lifecycle event and diagnostic log envelopes, run bounds and operation field rules. use pyo3::exceptions::PyTypeError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{ PyBool, PyByteArray, PyBytes, PyDict, PyDictMethods, PyInt, PyString, PyTypeMethods, }; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::lifecycle_events::constants::INVOCATION_SEQUENCE_PROBLEM; use sqlbuild_core::lifecycle_events::main::diagnostic_severity_problem::diagnostic_severity_problem; use sqlbuild_core::lifecycle_events::main::diagnostic_version_problem::diagnostic_version_problem; @@ -17,8 +17,8 @@ use sqlbuild_core::lifecycle_events::main::publishable_hook_name::publishable_ho use sqlbuild_core::lifecycle_events::main::run_bounds_problem::run_bounds_problem; use sqlbuild_core::lifecycle_events::models::{HookFacts, SequenceFact}; -use crate::bindings::_helpers::lifecycle_events::payloads::validation_error; -use crate::bindings::_helpers::lifecycle_events::validation::{ +use crate::lifecycle_events::_helpers::payloads::validation_error; +use crate::lifecycle_events::_helpers::validation::{ frozen_json, known_event_check, optional_text_check, required_text_check, schema_version_check, timestamp_check, }; @@ -104,7 +104,7 @@ fn freeze_object_field(owner: &Bound<'_, PyAny>, field: &str, path: &str) -> PyR /// Validate and freeze one lifecycle event envelope in place. #[pyfunction(name = "validate_lifecycle_event")] #[pyo3(signature = (*, event))] -fn lifecycle_event_check(event: &Bound<'_, PyAny>) -> PyResult<()> { +pub(crate) fn lifecycle_event_check(event: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = event.py(); for field in EVENT_REQUIRED_TEXT { required_text_check(&native_field(event, field)?, field)?; @@ -137,7 +137,7 @@ fn lifecycle_event_check(event: &Bound<'_, PyAny>) -> PyResult<()> { /// Validate a dispatcher-assigned invocation sequence. #[pyfunction(name = "validate_lifecycle_invocation_sequence")] #[pyo3(signature = (*, value))] -fn invocation_sequence_check(value: &Bound<'_, PyAny>) -> PyResult<()> { +pub(crate) fn invocation_sequence_check(value: &Bound<'_, PyAny>) -> PyResult<()> { if !nonnegative_int(value)? { return Err(validation_error(value.py(), INVOCATION_SEQUENCE_PROBLEM)); } @@ -147,7 +147,7 @@ fn invocation_sequence_check(value: &Bound<'_, PyAny>) -> PyResult<()> { /// Validate and freeze one diagnostic log record in place. #[pyfunction(name = "validate_diagnostic_log")] #[pyo3(signature = (*, record))] -fn diagnostic_log_check(record: &Bound<'_, PyAny>) -> PyResult<()> { +pub(crate) fn diagnostic_log_check(record: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = record.py(); let schema_version: Bound<'_, PyAny> = native_field(record, "schema_version")?; schema_version_check(&schema_version)?; @@ -171,7 +171,7 @@ fn diagnostic_log_check(record: &Bound<'_, PyAny>) -> PyResult<()> { /// Validate a run's selection, concurrency and worker counts. #[pyfunction(name = "validate_run_bounds")] #[pyo3(signature = (*, selected_count, configured_concurrency, worker_count))] -fn run_bounds_check( +pub(crate) fn run_bounds_check( selected_count: &Bound<'_, PyAny>, configured_concurrency: &Bound<'_, PyAny>, worker_count: &Bound<'_, PyAny>, @@ -203,7 +203,7 @@ fn run_bounds_check( /// The allow-listed, non-negative operation metadata counts. #[pyfunction] #[pyo3(signature = (*, metadata))] -fn validated_operation_metadata<'py>( +pub(crate) fn validated_operation_metadata<'py>( py: Python<'py>, metadata: &Bound<'py, PyAny>, ) -> PyResult> { @@ -251,7 +251,7 @@ fn hook_name_publishable(name: &Bound<'_, PyAny>) -> PyResult { /// The validated hook fields of an operation, empty when none are supplied. #[pyfunction] #[pyo3(signature = (*, hook_phase, hook_index, hook_type, hook_name))] -fn validated_operation_hook_fields<'py>( +pub(crate) fn validated_operation_hook_fields<'py>( hook_phase: &Bound<'py, PyAny>, hook_index: &Bound<'py, PyAny>, hook_type: &Bound<'py, PyAny>, @@ -287,7 +287,7 @@ fn validated_operation_hook_fields<'py>( /// Validate that a producer other than SQLBuild names its version. #[pyfunction(name = "validate_lifecycle_producer")] #[pyo3(signature = (*, producer, producer_version))] -fn producer_check( +pub(crate) fn producer_check( producer: &Bound<'_, PyAny>, producer_version: &Bound<'_, PyAny>, ) -> PyResult<()> { @@ -296,13 +296,3 @@ fn producer_check( None => Ok(()), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(lifecycle_event_check, module)?)?; - module.add_function(wrap_pyfunction!(invocation_sequence_check, module)?)?; - module.add_function(wrap_pyfunction!(diagnostic_log_check, module)?)?; - module.add_function(wrap_pyfunction!(run_bounds_check, module)?)?; - module.add_function(wrap_pyfunction!(validated_operation_metadata, module)?)?; - module.add_function(wrap_pyfunction!(validated_operation_hook_fields, module)?)?; - module.add_function(wrap_pyfunction!(producer_check, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/mod.rs new file mode 100644 index 0000000000..c387aad44f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod catalog; +pub(crate) mod envelopes; +pub(crate) mod payloads; +pub(crate) mod validation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/payloads.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs rename to crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/payloads.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs similarity index 86% rename from crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs rename to crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs index 8aaa1062e8..b2bc109d64 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs @@ -1,12 +1,12 @@ //! Envelope field validation, JSON freezing and catalogued event validation. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::{PyBool, PyInt, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::lifecycle_events::main::validate_known_event::validate_known_event; use sqlbuild_core::lifecycle_events::models::KnownEventFacts; -use crate::bindings::_helpers::lifecycle_events::payloads::{ +use crate::lifecycle_events::_helpers::payloads::{ PathSegment, frozen, object_entries, validation_error, }; use sqlbuild_python_base::boundary::main::imported::imported; @@ -101,11 +101,3 @@ pub(crate) fn known_event_check(event: &Bound<'_, PyAny>) -> PyResult<()> { }; validate_known_event(&facts).map_err(|message| validation_error(event.py(), &message)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(schema_version_check, module)?)?; - module.add_function(wrap_pyfunction!(frozen_json, module)?)?; - module.add_function(wrap_pyfunction!(required_text_check, module)?)?; - module.add_function(wrap_pyfunction!(optional_text_check, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/main/mod.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/main/mod.rs new file mode 100644 index 0000000000..2aaf10eb01 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/main/mod.rs @@ -0,0 +1,3 @@ +pub mod register_catalog; +pub mod register_envelopes; +pub mod register_validation; diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_catalog.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_catalog.rs new file mode 100644 index 0000000000..704fe846de --- /dev/null +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_catalog.rs @@ -0,0 +1,49 @@ +//! Register the catalog functions and classes. + +use crate::lifecycle_events::_helpers::catalog::lifecycle_event_catalogued; +use crate::lifecycle_events::_helpers::catalog::lifecycle_event_terminal; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyFrozenSet; +use pyo3::wrap_pyfunction; +use sqlbuild_core::lifecycle_events::constants::{ + CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, HOOK_PHASES, + HOOK_TYPES, LIFECYCLE_EVENT_SCHEMA_VERSIONS, OPERATION_ADAPTERS, OPERATION_KINDS, + OPERATION_METADATA_FIELDS, OPERATION_NAMES, OPERATION_PHASES, OPERATION_SCOPES, + OPERATION_STRATEGIES, OPERATION_TARGET_KINDS, RESOURCE_SKIP_CODES, RESOURCE_SKIP_MODES, +}; + +/// Register the catalog functions and classes. +pub fn register_catalog(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py = module.py(); + module.add( + "CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION", + CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, + )?; + module.add( + "CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION", + CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, + )?; + module.add( + "LIFECYCLE_EVENT_SCHEMA_VERSIONS", + PyFrozenSet::new(py, LIFECYCLE_EVENT_SCHEMA_VERSIONS)?, + )?; + let sets: [(&str, &[&str]); 12] = [ + ("HOOK_PHASES", &HOOK_PHASES), + ("HOOK_TYPES", &HOOK_TYPES), + ("OPERATION_ADAPTERS", &OPERATION_ADAPTERS), + ("OPERATION_KINDS", &OPERATION_KINDS), + ("OPERATION_METADATA_FIELDS", &OPERATION_METADATA_FIELDS), + ("OPERATION_NAMES", &OPERATION_NAMES), + ("OPERATION_PHASES", &OPERATION_PHASES), + ("OPERATION_SCOPES", &OPERATION_SCOPES), + ("OPERATION_STRATEGIES", &OPERATION_STRATEGIES), + ("OPERATION_TARGET_KINDS", &OPERATION_TARGET_KINDS), + ("RESOURCE_SKIP_CODES", &RESOURCE_SKIP_CODES), + ("RESOURCE_SKIP_MODES", &RESOURCE_SKIP_MODES), + ]; + for (name, values) in sets { + module.add(name, PyFrozenSet::new(py, values)?)?; + } + module.add_function(wrap_pyfunction!(lifecycle_event_catalogued, module)?)?; + module.add_function(wrap_pyfunction!(lifecycle_event_terminal, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_envelopes.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_envelopes.rs new file mode 100644 index 0000000000..7b902515f6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_envelopes.rs @@ -0,0 +1,22 @@ +//! Register the envelopes functions and classes. + +use crate::lifecycle_events::_helpers::envelopes::diagnostic_log_check; +use crate::lifecycle_events::_helpers::envelopes::invocation_sequence_check; +use crate::lifecycle_events::_helpers::envelopes::lifecycle_event_check; +use crate::lifecycle_events::_helpers::envelopes::producer_check; +use crate::lifecycle_events::_helpers::envelopes::run_bounds_check; +use crate::lifecycle_events::_helpers::envelopes::validated_operation_hook_fields; +use crate::lifecycle_events::_helpers::envelopes::validated_operation_metadata; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the envelopes functions and classes. +pub fn register_envelopes(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(lifecycle_event_check, module)?)?; + module.add_function(wrap_pyfunction!(invocation_sequence_check, module)?)?; + module.add_function(wrap_pyfunction!(diagnostic_log_check, module)?)?; + module.add_function(wrap_pyfunction!(run_bounds_check, module)?)?; + module.add_function(wrap_pyfunction!(validated_operation_metadata, module)?)?; + module.add_function(wrap_pyfunction!(validated_operation_hook_fields, module)?)?; + module.add_function(wrap_pyfunction!(producer_check, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs new file mode 100644 index 0000000000..40cd95ae55 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs @@ -0,0 +1,17 @@ +//! Register the validation functions and classes. + +use crate::lifecycle_events::_helpers::validation::frozen_json; +use crate::lifecycle_events::_helpers::validation::optional_text_check; +use crate::lifecycle_events::_helpers::validation::required_text_check; +use crate::lifecycle_events::_helpers::validation::schema_version_check; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the validation functions and classes. +pub fn register_validation(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(schema_version_check, module)?)?; + module.add_function(wrap_pyfunction!(frozen_json, module)?)?; + module.add_function(wrap_pyfunction!(required_text_check, module)?)?; + module.add_function(wrap_pyfunction!(optional_text_check, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-planner/src/lifecycle_events/mod.rs b/crates/sqlbuild-python-planner/src/lifecycle_events/mod.rs new file mode 100644 index 0000000000..e083d0b80f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/lifecycle_events/mod.rs @@ -0,0 +1,4 @@ +//! The lifecycle event catalogue, envelopes, payload values and validation. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs b/crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs rename to crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs index a6ed3997df..6ece8f8a74 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs +++ b/crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs @@ -32,11 +32,6 @@ pub(crate) fn optional_text(value: &Bound<'_, PyAny>) -> PyResult Ok(Some(value.extract()?)) } -/// `value.timestamp()` of a `datetime`. -pub(crate) fn timestamp(value: &Bound<'_, PyAny>) -> PyResult { - value.call_method0("timestamp")?.extract() -} - /// Raw cursor bounds of one cursor type, parsed once per distinct text. pub(crate) struct BoundParser<'py> { session: CursorSession<'py>, diff --git a/crates/sqlbuild-python-planner/src/microbatches/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/microbatches/_helpers/mod.rs new file mode 100644 index 0000000000..dc529131de --- /dev/null +++ b/crates/sqlbuild-python-planner/src/microbatches/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod conversion; +pub(crate) mod projection; +pub(crate) mod statements; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs b/crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs rename to crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs index 1ebcc33d77..1323a18b41 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs +++ b/crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs @@ -1,8 +1,9 @@ //! Coverage, replay-requirement and latest-requirement projections over append-only events. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::microbatches::main::timestamp::timestamp; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_state::microbatches::main::latest_requirement::latest_requirement; use sqlbuild_state::microbatches::main::project_coverage::project_coverage; @@ -12,9 +13,7 @@ use sqlbuild_state::microbatches::models::{ ReplayState, ReplayVersions, RequirementFact, TextInterval, }; -use crate::bindings::_helpers::microbatches::conversion::{ - BoundParser, enum_text, optional_text, timestamp, -}; +use crate::microbatches::_helpers::conversion::{BoundParser, enum_text, optional_text}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -107,7 +106,7 @@ fn python_coverage<'py>( /// Fold completion facts and classify uncovered expected intervals. #[pyfunction(name = "project_microbatch_coverage")] #[pyo3(signature = (*, events, expected_intervals, cursor_type))] -fn coverage<'py>( +pub(crate) fn coverage<'py>( events: &Bound<'py, PyAny>, expected_intervals: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, @@ -165,7 +164,7 @@ fn replay_state_member(state: ReplayState) -> &'static str { /// Derive replay completion without writing mutable requirement status. #[pyfunction(name = "project_replay_requirement")] #[pyo3(signature = (**request))] -fn replay<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn replay<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { let request = Request::new("project_replay_requirement", request)?; let py: Python<'py> = request.py(); let requirement: Bound<'py, PyAny> = request.get("requirement")?; @@ -225,7 +224,7 @@ fn replay<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult( +pub(crate) fn latest<'py>( events: &Bound<'py, PyAny>, current_model_version_hash: &str, ) -> PyResult>> { @@ -251,9 +250,3 @@ fn latest<'py>( } Ok(latest_requirement(&facts, current_model_version_hash).map(|index| objects[index].clone())) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(coverage, module)?)?; - module.add_function(wrap_pyfunction!(replay, module)?)?; - module.add_function(wrap_pyfunction!(latest, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs b/crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs rename to crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs index dee8a8be60..688f4379bb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs +++ b/crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs @@ -1,8 +1,8 @@ //! Direct-mode microbatch state DDL, idempotent inserts, reads and event IDs. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::MICROBATCH_TABLE_NAME; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; use sqlbuild_state::events::main::microbatch_event_id::microbatch_event_id; @@ -14,7 +14,7 @@ use sqlbuild_state::microbatches::models::{MicrobatchEventIdentity, ScopeRead}; use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; use sqlbuild_state::state_tables::models::TableColumn; -use crate::bindings::_helpers::microbatches::conversion::{optional_text, state_error}; +use crate::microbatches::_helpers::conversion::{optional_text, state_error}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::contracts::main::render_state_sql_literal::render_state_sql_literal; @@ -127,7 +127,7 @@ fn events_table<'py>( /// Build one guarded set-based insert for events in the same state table. #[pyfunction(name = "build_microbatch_insert_many_sql")] #[pyo3(signature = (*, events, render_qualified_name, timestamp_type = None))] -fn insert_many( +pub(crate) fn insert_many( events: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, timestamp_type: Option<&str>, @@ -144,7 +144,7 @@ fn insert_many( /// Build one lookup for event IDs already present before a bulk insert. #[pyfunction(name = "build_microbatch_existing_event_ids_sql")] #[pyo3(signature = (*, events, render_qualified_name))] -fn existing_ids( +pub(crate) fn existing_ids( events: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, ) -> PyResult { @@ -160,7 +160,7 @@ fn existing_ids( /// Build the ordered read of one scope's events, every generation for the wildcard. #[pyfunction(name = "build_microbatch_read_scope_sql")] #[pyo3(signature = (*, scope, render_qualified_name))] -fn read_scope( +pub(crate) fn read_scope( scope: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, ) -> PyResult { @@ -190,7 +190,7 @@ fn read_scope( /// Build direct microbatch-state DDL with the adapter's text and timestamp types. #[pyfunction(name = "build_microbatch_create_table_sql")] #[pyo3(signature = (**request))] -fn create_table(request: Option<&Bound<'_, PyDict>>) -> PyResult { +pub(crate) fn create_table(request: Option<&Bound<'_, PyDict>>) -> PyResult { let request = Request::new("build_microbatch_create_table_sql", request)?; let py: Python<'_> = request.py(); let table: String = qualified_table( @@ -227,7 +227,9 @@ fn create_table(request: Option<&Bound<'_, PyDict>>) -> PyResult { /// Build non-unique scope and provenance indexes for indexed warehouses. #[pyfunction(name = "build_microbatch_create_index_sqls")] #[pyo3(signature = (**request))] -fn create_indexes<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn create_indexes<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("build_microbatch_create_index_sqls", request)?; let py: Python<'_> = request.py(); let schema: Bound<'_, PyAny> = request.get("schema")?; @@ -264,7 +266,7 @@ fn create_indexes<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult>) -> PyResult { +pub(crate) fn deterministic_event_id(request: Option<&Bound<'_, PyDict>>) -> PyResult { let request = Request::new("deterministic_microbatch_event_id", request)?; let scope: Bound<'_, PyAny> = request.get("scope")?; let field = |name: &str| -> PyResult { native_field(&scope, name)?.extract() }; @@ -285,12 +287,3 @@ fn deterministic_event_id(request: Option<&Bound<'_, PyDict>>) -> PyResult) -> PyResult<()> { - module.add_function(wrap_pyfunction!(insert_many, module)?)?; - module.add_function(wrap_pyfunction!(existing_ids, module)?)?; - module.add_function(wrap_pyfunction!(read_scope, module)?)?; - module.add_function(wrap_pyfunction!(create_table, module)?)?; - module.add_function(wrap_pyfunction!(create_indexes, module)?)?; - module.add_function(wrap_pyfunction!(deterministic_event_id, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/microbatches/main/mod.rs b/crates/sqlbuild-python-planner/src/microbatches/main/mod.rs new file mode 100644 index 0000000000..a06eec5634 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/microbatches/main/mod.rs @@ -0,0 +1,3 @@ +pub mod register_projection; +pub mod register_statements; +pub mod timestamp; diff --git a/crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs b/crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs new file mode 100644 index 0000000000..f10c287d0c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs @@ -0,0 +1,14 @@ +//! Register the projection functions and classes. + +use crate::microbatches::_helpers::projection::coverage; +use crate::microbatches::_helpers::projection::latest; +use crate::microbatches::_helpers::projection::replay; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the projection functions and classes. +pub fn register_projection(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(coverage, module)?)?; + module.add_function(wrap_pyfunction!(replay, module)?)?; + module.add_function(wrap_pyfunction!(latest, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/microbatches/main/register_statements.rs b/crates/sqlbuild-python-planner/src/microbatches/main/register_statements.rs new file mode 100644 index 0000000000..4e10e3c4a3 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/microbatches/main/register_statements.rs @@ -0,0 +1,20 @@ +//! Register the statements functions and classes. + +use crate::microbatches::_helpers::statements::create_indexes; +use crate::microbatches::_helpers::statements::create_table; +use crate::microbatches::_helpers::statements::deterministic_event_id; +use crate::microbatches::_helpers::statements::existing_ids; +use crate::microbatches::_helpers::statements::insert_many; +use crate::microbatches::_helpers::statements::read_scope; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the statements functions and classes. +pub fn register_statements(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(insert_many, module)?)?; + module.add_function(wrap_pyfunction!(existing_ids, module)?)?; + module.add_function(wrap_pyfunction!(read_scope, module)?)?; + module.add_function(wrap_pyfunction!(create_table, module)?)?; + module.add_function(wrap_pyfunction!(create_indexes, module)?)?; + module.add_function(wrap_pyfunction!(deterministic_event_id, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/microbatches/main/timestamp.rs b/crates/sqlbuild-python-planner/src/microbatches/main/timestamp.rs new file mode 100644 index 0000000000..d73192a825 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/microbatches/main/timestamp.rs @@ -0,0 +1,8 @@ +//! `value.timestamp()` of a `datetime`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +/// `value.timestamp()` of a `datetime`. +pub fn timestamp(value: &Bound<'_, PyAny>) -> PyResult { + value.call_method0("timestamp")?.extract() +} diff --git a/crates/sqlbuild-python-planner/src/microbatches/mod.rs b/crates/sqlbuild-python-planner/src/microbatches/mod.rs new file mode 100644 index 0000000000..7ba5cc3b54 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/microbatches/mod.rs @@ -0,0 +1,4 @@ +//! Microbatch history: events read from Python, their projections and the direct-mode state statements. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/mod.rs b/crates/sqlbuild-python-planner/src/observations/_helpers/mod.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/observations/mod.rs rename to crates/sqlbuild-python-planner/src/observations/_helpers/mod.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs b/crates/sqlbuild-python-planner/src/observations/_helpers/publication.rs similarity index 79% rename from crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs rename to crates/sqlbuild-python-planner/src/observations/_helpers/publication.rs index 84d46233af..f1971faabe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs +++ b/crates/sqlbuild-python-planner/src/observations/_helpers/publication.rs @@ -8,16 +8,16 @@ use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_quer use sqlbuild_analysis::planner_warehouse::models::CursorTagRequest; use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND}; -use crate::bindings::_helpers::consumption::relation_reads::relation; -use crate::bindings::_helpers::observations::source_capture::take_source_capture; -use crate::bindings::_helpers::planner_warehouse::constants::RUN_BOUNDED; -use crate::bindings::_helpers::planner_warehouse::cursors::{adapter_cursor_sql, bound_task}; +use crate::consumption::main::relation::relation; +use crate::planner_warehouse::constants::RUN_BOUNDED; +use crate::planner_warehouse::main::inspection::adapter_cursor_sql::adapter_cursor_sql; +use crate::planner_warehouse::main::inspection::bound_task::bound_task; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::native_host::main::append_observations::append_observations; +use sqlbuild_python_front::native_host::models::Location; use sqlbuild_python_front::native_host::models::ObservationAppend; -use sqlbuild_python_front::native_host::models::{ConsumptionConnection, Location}; use sqlbuild_python_project::compiled_views::main::compiled_models::dep_rows::dep_rows; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; @@ -29,7 +29,7 @@ const SOURCE_DEPENDENCY: &str = "source"; const MAX_OBSERVATION_READS: i64 = 4; /// One successful incremental model's cursor read: its name, state location, cursor type and SQL. -struct CursorRead { +pub(crate) struct CursorRead { model: String, location: Location, cursor_type: String, @@ -37,7 +37,7 @@ struct CursorRead { } /// The cursor read of every successful incremental model, rendered as the planner renders it. -fn cursor_reads( +pub(crate) fn cursor_reads( project: &Bound<'_, PyAny>, adapter: &Bound<'_, PyAny>, successful_models: &[String], @@ -82,7 +82,7 @@ fn cursor_reads( } /// One `source_columns` row per described source, in the state schema of each model reading it. -fn source_column_rows( +pub(crate) fn source_column_rows( project: &Bound<'_, PyAny>, described: &BTreeMap, run_id: &str, @@ -126,7 +126,7 @@ fn source_column_rows( } /// Read each cursor `MAX`, then append it with `rows`, by state location; one warning per failure. -fn observe<'py>( +pub(crate) fn observe<'py>( reads: &[CursorRead], mut rows: BTreeMap>, (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -185,34 +185,3 @@ fn observe<'py>( } Ok(warnings) } - -/// Record successful incremental models' cursor `MAX` and the source columns planning described; failures only warn. -pub(crate) fn append_build_observations( - (project, successful_models): (&Bound<'_, PyAny>, &[String]), - (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - run_id: &str, -) -> Vec { - let described: BTreeMap = take_source_capture().into_iter().collect(); - let recorded: PyResult> = (|| { - let reads: Vec = cursor_reads(project, adapter, successful_models)?; - let sources: BTreeMap> = - source_column_rows(project, &described, run_id)?; - if reads.is_empty() && sources.is_empty() { - return Ok(Vec::new()); - } - let py: Python<'_> = adapter.py(); - let connection = ConsumptionConnection::new(adapter, connection_config); - let observed: PyResult> = connection - .connected(py) - .and_then(|handle| observe(&reads, sources, (adapter, &handle), run_id)); - let closed: PyResult<()> = connection.close(py); - let warnings: Vec = observed?; - closed?; - Ok(warnings) - })(); - recorded.unwrap_or_else(|error| { - vec![format!( - "could not record observations: {error}; plan previews show them as unknown" - )] - }) -} diff --git a/crates/sqlbuild-python-planner/src/observations/_helpers/source_capture.rs b/crates/sqlbuild-python-planner/src/observations/_helpers/source_capture.rs new file mode 100644 index 0000000000..78ca319182 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/observations/_helpers/source_capture.rs @@ -0,0 +1,11 @@ +//! The expression-source columns a build's planning described, kept for its observations. + +use std::cell::RefCell; + +/// Each described source's name and observation value, in the order planning described them. +pub(crate) type DescribedSources = Vec<(String, String)>; + +thread_local! { + /// Per thread, since a build plans and records on the thread that runs it; `None` when not armed. + pub(crate) static CAPTURED: RefCell> = const { RefCell::new(None) }; +} diff --git a/crates/sqlbuild-python-planner/src/observations/main/append_build_observations.rs b/crates/sqlbuild-python-planner/src/observations/main/append_build_observations.rs new file mode 100644 index 0000000000..d0e653af69 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/observations/main/append_build_observations.rs @@ -0,0 +1,42 @@ +//! Record successful incremental models' cursor `MAX` and the source columns planning described; failures only warn. + +use crate::observations::_helpers::publication::CursorRead; +use crate::observations::_helpers::publication::cursor_reads; +use crate::observations::_helpers::publication::observe; +use crate::observations::_helpers::publication::source_column_rows; +use crate::observations::main::take_source_capture::take_source_capture; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_front::native_host::models::ObservationAppend; +use sqlbuild_python_front::native_host::models::{ConsumptionConnection, Location}; +use std::collections::BTreeMap; + +/// Record successful incremental models' cursor `MAX` and the source columns planning described; failures only warn. +pub fn append_build_observations( + (project, successful_models): (&Bound<'_, PyAny>, &[String]), + (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + run_id: &str, +) -> Vec { + let described: BTreeMap = take_source_capture().into_iter().collect(); + let recorded: PyResult> = (|| { + let reads: Vec = cursor_reads(project, adapter, successful_models)?; + let sources: BTreeMap> = + source_column_rows(project, &described, run_id)?; + if reads.is_empty() && sources.is_empty() { + return Ok(Vec::new()); + } + let py: Python<'_> = adapter.py(); + let connection = ConsumptionConnection::new(adapter, connection_config); + let observed: PyResult> = connection + .connected(py) + .and_then(|handle| observe(&reads, sources, (adapter, &handle), run_id)); + let closed: PyResult<()> = connection.close(py); + let warnings: Vec = observed?; + closed?; + Ok(warnings) + })(); + recorded.unwrap_or_else(|error| { + vec![format!( + "could not record observations: {error}; plan previews show them as unknown" + )] + }) +} diff --git a/crates/sqlbuild-python-planner/src/observations/main/arm_source_capture.rs b/crates/sqlbuild-python-planner/src/observations/main/arm_source_capture.rs new file mode 100644 index 0000000000..df8a0a1959 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/observations/main/arm_source_capture.rs @@ -0,0 +1,8 @@ +//! Start capturing described source columns; only a build arms it, so `sqb plan` records nothing. + +use crate::observations::_helpers::source_capture::CAPTURED; + +/// Start capturing described source columns; only a build arms it, so `sqb plan` records nothing. +pub fn arm_source_capture() { + CAPTURED.with(|slot| *slot.borrow_mut() = Some(Vec::new())); +} diff --git a/crates/sqlbuild-python-planner/src/observations/main/capture_source_columns.rs b/crates/sqlbuild-python-planner/src/observations/main/capture_source_columns.rs new file mode 100644 index 0000000000..e854c03d54 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/observations/main/capture_source_columns.rs @@ -0,0 +1,26 @@ +//! Keep one source's described SQL hash and column names when a build armed the capture. + +use crate::observations::_helpers::source_capture::CAPTURED; +use serde_json::{Map, Value}; +use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; +use sqlbuild_state::observations::constants::{ + SOURCE_COLUMNS_NAMES_KEY, SOURCE_COLUMNS_SQL_HASH_KEY, +}; + +/// Keep one source's described SQL hash and column names when a build armed the capture. +pub fn capture_source_columns(source: &str, sql: &str, columns: &[String]) { + CAPTURED.with(|slot| { + if let Some(captured) = slot.borrow_mut().as_mut() { + let mut value: Map = Map::new(); + value.insert( + SOURCE_COLUMNS_SQL_HASH_KEY.to_owned(), + Value::String(sha256_hex(sql.as_bytes())), + ); + value.insert( + SOURCE_COLUMNS_NAMES_KEY.to_owned(), + Value::from(columns.to_vec()), + ); + captured.push((source.to_owned(), Value::Object(value).to_string())); + } + }); +} diff --git a/crates/sqlbuild-python-planner/src/observations/main/mod.rs b/crates/sqlbuild-python-planner/src/observations/main/mod.rs new file mode 100644 index 0000000000..9919a39848 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/observations/main/mod.rs @@ -0,0 +1,4 @@ +pub mod append_build_observations; +pub mod arm_source_capture; +pub mod capture_source_columns; +pub mod take_source_capture; diff --git a/crates/sqlbuild-python-planner/src/observations/main/take_source_capture.rs b/crates/sqlbuild-python-planner/src/observations/main/take_source_capture.rs new file mode 100644 index 0000000000..5e0175e45d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/observations/main/take_source_capture.rs @@ -0,0 +1,11 @@ +//! The captured source columns, disarming the capture. + +use crate::observations::_helpers::source_capture::CAPTURED; +use crate::observations::_helpers::source_capture::DescribedSources; + +/// The captured source columns, disarming the capture. +pub fn take_source_capture() -> DescribedSources { + CAPTURED + .with(|slot| slot.borrow_mut().take()) + .unwrap_or_default() +} diff --git a/crates/sqlbuild-python-planner/src/observations/mod.rs b/crates/sqlbuild-python-planner/src/observations/mod.rs new file mode 100644 index 0000000000..c86c2d4322 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/observations/mod.rs @@ -0,0 +1,4 @@ +//! What a build observed, cursor maximums and described source columns, for plan previews. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs rename to crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs index bef1e1810a..6f1ef98fbb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs @@ -1,12 +1,12 @@ //! One planned model's requested, resolved and deferred cursor state, for text and JSON plans. +use crate::plan_output::main::aligned_timestamp_bounds::aligned_timestamp_bounds; +use crate::plan_output::main::resolve_effective_microbatch_batch_size::resolve_effective_microbatch_batch_size; +use crate::plan_output::main::text::text; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::plan_output::display::{ - aligned_timestamp_bounds, resolve_effective_microbatch_batch_size, -}; -use crate::bindings::_helpers::plan_output::values::{optional_text, text}; +use crate::plan_output::_helpers::values::optional_text; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs new file mode 100644 index 0000000000..962c5d026d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs @@ -0,0 +1,131 @@ +//! Plan and progress display fragments over planner entries and cursor values. + +use crate::plan_output::main::text::text; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyString}; +use sqlbuild_core::plan_display::constants::{ + DEFAULT_CURSOR_GRAIN, INTEGER_CURSOR, TIMESTAMP_CURSOR, +}; +use sqlbuild_core::plan_display::main::format_no_input_rows::format_no_input_rows as no_input_rows; +use sqlbuild_core::plan_display::main::format_waiting_on_empty_inputs::format_waiting_on_empty_inputs as waiting_on_empty_inputs; +use sqlbuild_core::plan_display::main::model_materialization_label::model_materialization_label as materialization_label; +use sqlbuild_core::plan_display::models::{ModelLabelFacts, ParsedBound}; + +use crate::plan_output::_helpers::values::optional_text; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +const DATETIME_MODULE: &str = "datetime"; + +pub(crate) fn label_facts(entry: &Bound<'_, PyAny>) -> PyResult { + let field = + |name: &str| -> PyResult> { optional_text(&native_field(entry, name)?) }; + Ok(ModelLabelFacts { + materialization_type: text(&native_field(entry, "materialization_type")?)?, + action: text(&native_field(entry, "action")?)?, + custom_materialization_name: field("custom_materialization_name")?, + incremental_strategy: field("incremental_strategy")?, + incremental_mode: field("incremental_mode")?, + cursor_type: field("cursor_type")?, + snapshot_strategy: field("snapshot_strategy")?, + historical: !native_field(entry, "observed_at_column")?.is_none(), + historical_input: field("historical_input")?, + }) +} + +/// The cursor type, defaulting to timestamp, and its grain object (`None` for integers). +pub(crate) fn cursor_typing<'py>( + cursor_type: &Bound<'py, PyAny>, + cursor_grain: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = cursor_type.py(); + let effective: Bound<'py, PyAny> = if cursor_type.is_truthy()? { + cursor_type.clone() + } else { + PyString::new(py, TIMESTAMP_CURSOR).into_any() + }; + if effective.eq(INTEGER_CURSOR)? { + return Ok((effective, py.None().into_bound(py))); + } + let grain: Bound<'py, PyAny> = if cursor_grain.is_truthy()? { + cursor_grain.clone() + } else { + PyString::new(py, DEFAULT_CURSOR_GRAIN).into_any() + }; + let grain: Bound<'py, PyAny> = enum_from_value(py, RecordId::CursorGrain, grain)?; + Ok((effective, grain)) +} + +pub(crate) fn parsed_bound(value: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = value.py(); + let parsed: Bound<'_, PyAny> = + match imported(py, DATETIME_MODULE, "datetime")?.call_method1("fromisoformat", (value,)) { + Ok(parsed) => parsed, + Err(error) if error.is_instance_of::(py) => return Ok(None), + Err(error) => return Err(error), + }; + let mut midnight: bool = true; + for part in ["hour", "minute", "second", "microsecond"] { + midnight = midnight && native_field(&parsed, part)?.extract::()? == 0; + } + Ok(Some(ParsedBound { + date: parsed + .call_method0("date")? + .call_method0("isoformat")? + .extract()?, + midnight, + })) +} + +/// The full materialization label used in plan summaries. +pub(crate) fn model_materialization_label(entry: &Bound<'_, PyAny>) -> PyResult { + Ok(materialization_label(&label_facts(entry)?)) +} + +fn names(input_names: &Bound<'_, PyAny>) -> PyResult> { + input_names + .try_iter()? + .map(|name| name?.extract()) + .collect() +} + +/// Explain that a cursor window is empty because its inputs have no rows. +pub(crate) fn format_no_input_rows(input_names: &Bound<'_, PyAny>) -> PyResult { + Ok(no_input_rows(&names(input_names)?)) +} + +/// Explain that a cursor window waits on empty inputs while other inputs have rows. +#[pyfunction] +pub(crate) fn format_waiting_on_empty_inputs(input_names: &Bound<'_, PyAny>) -> PyResult { + Ok(waiting_on_empty_inputs(&names(input_names)?)) +} + +pub(crate) fn as_datetime<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let inner: Bound<'py, PyAny> = native_field(value, "value")?; + if value.is_instance(&record_class(py, RecordId::TimestampValue)?)? { + return Ok(inner); + } + let datetime: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; + let midnight: Bound<'py, PyAny> = native_field(&datetime, "min")?.call_method0("time")?; + datetime.call_method1("combine", (inner, midnight)) +} + +pub(crate) fn presume_utc<'py>(value: Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if !native_field(&value, "tzinfo")?.is_none() { + return Ok(value); + } + let options: Bound<'py, PyDict> = PyDict::new(py); + pyo3::types::PyDictMethods::set_item( + &options, + "tzinfo", + imported(py, DATETIME_MODULE, "UTC")?, + )?; + value.call_method("replace", (), Some(&options)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs rename to crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs index 606fc94820..b8bc601539 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs @@ -1,11 +1,11 @@ //! Convert public migration models at the Python boundary for native presentation. -use crate::bindings::_helpers::plan_output::plan_text::display_limit; -use crate::bindings::_helpers::plan_output::values::{ - iso, items, optional_text, relation_name, text, texts, -}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::plan_output::_helpers::plan_text::display_limit; +use crate::plan_output::_helpers::values::{iso, optional_text, relation_name, texts}; +use crate::plan_output::main::items::items; +use crate::plan_output::main::text::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use sqlbuild_core::cli_text::models::{ CliStyle, ColumnMigrationText, MigrationDecisionText, MigrationText, MigrationTextReport, OldNameText, PreBuildWorkText, RetentionText, TableTypeText, @@ -185,7 +185,7 @@ pub(crate) fn pre_build_lines( /// Format retention, table-type, and migration work that runs before model builds. #[pyfunction(name = "format_pre_build_work", signature = (*, lines, plan, display_options))] -fn format_pre_build( +pub(crate) fn format_pre_build( lines: Vec, plan: &Bound<'_, PyAny>, display_options: &Bound<'_, PyAny>, @@ -194,7 +194,10 @@ fn format_pre_build( } #[pyfunction(name = "old_name_rows", signature = (*, entry, style))] -fn render_old_name(entry: &Bound<'_, PyAny>, style: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn render_old_name( + entry: &Bound<'_, PyAny>, + style: &Bound<'_, PyAny>, +) -> PyResult> { Ok(old_name_rows::old_name_rows( &old_name(entry)?, CliStyle { @@ -202,8 +205,3 @@ fn render_old_name(entry: &Bound<'_, PyAny>, style: &Bound<'_, PyAny>) -> PyResu }, )) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(format_pre_build, module)?)?; - module.add_function(wrap_pyfunction!(render_old_name, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/mod.rs new file mode 100644 index 0000000000..3f6b3b4a7e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod cursor_details; +pub(crate) mod display; +pub(crate) mod migrations; +pub(crate) mod plan_json; +pub(crate) mod plan_text; +pub(crate) mod python_identity; +pub(crate) mod safety; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs similarity index 83% rename from crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs rename to crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs index 983a0772dc..58f9c36c29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs @@ -1,28 +1,25 @@ //! The JSON plan document, read from the planner's output by attribute. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::plan_output::main::any::any; +use crate::plan_output::main::dict::dict; +use crate::plan_output::main::future_cursor_safety::future_cursor_safety; +use crate::plan_output::main::items::items; +use crate::plan_output::main::maximum_start_safety::maximum_start_safety; +use crate::plan_output::main::text::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::plan_display::constants::SYNCED_COPY_KEY; -use crate::bindings::_helpers::plan_output::cursor_details::{ - CursorDetails, cursor_details, owner, -}; -use crate::bindings::_helpers::plan_output::plan_text::{model_config_diff, model_query_diff}; -use crate::bindings::_helpers::plan_output::python_identity::{ +use crate::plan_output::_helpers::cursor_details::{CursorDetails, cursor_details, owner}; +use crate::plan_output::_helpers::plan_text::{model_config_diff, model_query_diff}; +use crate::plan_output::_helpers::python_identity::{ definition_source_text, dependency_source_text, }; -use crate::bindings::_helpers::plan_output::safety::{future_cursor_safety, maximum_start_safety}; -use crate::bindings::_helpers::plan_output::values::{ - any, dict, enum_value, iso, items, list, mapped, optional_enum_value, relation_name, text, +use crate::plan_output::_helpers::values::{ + enum_value, iso, list, mapped, optional_enum_value, relation_name, }; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::compile_outputs::main::indented_json::indented_json; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; -/// The plan JSON key of what `--changed` would run, hold back and skip. -pub(crate) const CHANGED_KEY: &str = "changed"; const MICROBATCH_MODE: &str = "microbatch"; const RENAMED_REASON: &str = "renamed"; const RENAMED_DECISION: &str = "renamed"; @@ -148,7 +145,7 @@ fn microbatch_state<'py>(entry: &Bound<'py, PyAny>) -> PyResult(entry: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn model_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = entry.py(); let model: Bound<'py, PyDict> = dict( py, @@ -296,7 +293,7 @@ fn with_qualified_name<'py>( Ok(item.into_any()) } -fn seed_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn seed_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = entry.py(); with_qualified_name( dict( @@ -310,7 +307,7 @@ fn seed_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { ) } -fn source_load_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn source_load_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = entry.py(); let load: Bound<'py, PyDict> = dict( py, @@ -337,7 +334,7 @@ fn source_load_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult(entry: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn function_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = entry.py(); let columns: Bound<'py, PyAny> = native_field(entry, "return_columns")?; with_qualified_name( @@ -381,7 +378,7 @@ fn function_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> ) } -fn warning_json<'py>(warning: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn warning_json<'py>(warning: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = warning.py(); let item: Bound<'py, PyDict> = dict( py, @@ -454,7 +451,7 @@ fn identity_diff<'py>(entry: &Bound<'py, PyAny>) -> PyResult> Ok(result) } -fn python_entry_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn python_entry_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = entry.py(); let item: Bound<'py, PyDict> = dict( py, @@ -480,7 +477,7 @@ struct Usage<'py> { module: Bound<'py, PyAny>, } -fn providers_json<'py>( +pub(crate) fn providers_json<'py>( plan: &Bound<'py, PyAny>, python_entries: &[Bound<'py, PyAny>], ) -> PyResult> { @@ -558,7 +555,7 @@ fn providers_json<'py>( Ok(providers.into_any()) } -fn old_name_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn old_name_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = entry.py(); Ok(dict( py, @@ -584,7 +581,7 @@ fn old_name_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> .into_any()) } -fn migrations_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn migrations_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = plan.py(); let old_names: Bound<'py, PyDict> = PyDict::new(py); for entry in items(&native_field(plan, "old_name_view_entries")?)? { @@ -660,7 +657,7 @@ fn migrations_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> }) } -fn column_migrations_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn column_migrations_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = plan.py(); mapped(&native_field(plan, "column_migration_entries")?, |entry| { Ok(dict( @@ -687,7 +684,7 @@ fn column_migrations_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult(plan: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn retention_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = plan.py(); mapped(&native_field(plan, "retention_entries")?, |entry| { let request: Bound<'py, PyAny> = native_field(entry, "request")?; @@ -717,7 +714,7 @@ fn retention_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> }) } -fn table_type_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn table_type_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = plan.py(); mapped(&native_field(plan, "table_type_entries")?, |entry| { Ok(dict( @@ -739,103 +736,3 @@ fn table_type_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> .into_any()) }) } - -/// The plan as one JSON document, `json.dumps(indent=2)`. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn format_plan_json( - py: Python<'_>, - options: Option>, -) -> PyResult { - let arguments: KeywordArguments<'_> = KeywordArguments::read( - py, - "format_plan_json", - options, - ( - &["plan"], - &["python_plan_entries", SYNCED_COPY_KEY, CHANGED_KEY], - ), - )?; - let plan: Bound<'_, PyAny> = arguments.value("plan")?; - let python_entries: Vec> = if arguments.passed("python_plan_entries")? { - items(&arguments.value("python_plan_entries")?)? - } else { - Vec::new() - }; - let length = |field: &str| -> PyResult { native_field(&plan, field)?.len() }; - let python_nodes: Vec> = python_entries - .iter() - .map(python_entry_json) - .collect::>()?; - let result: Bound<'_, PyDict> = dict( - py, - vec![ - ( - "selected_count", - any( - py, - length("model_entries")? - + length("seed_entries")? - + length("function_entries")?, - )?, - ), - ( - "source_load_count", - any(py, length("source_load_entries")?)?, - ), - ("python_node_count", any(py, python_nodes.len())?), - ( - "models", - mapped(&native_field(&plan, "model_entries")?, model_json)?, - ), - ( - "seeds", - mapped(&native_field(&plan, "seed_entries")?, seed_json)?, - ), - ( - "source_loads", - mapped( - &native_field(&plan, "source_load_entries")?, - source_load_json, - )?, - ), - ( - "functions", - mapped(&native_field(&plan, "function_entries")?, function_json)?, - ), - ("python_nodes", PyList::new(py, python_nodes)?.into_any()), - ("providers", providers_json(&plan, &python_entries)?), - ( - "warnings", - mapped(&native_field(&plan, "warnings")?, warning_json)?, - ), - ("retention", retention_json(&plan)?), - ("table_type_conversions", table_type_json(&plan)?), - ("migrations", migrations_json(&plan)?), - ("column_migrations", column_migrations_json(&plan)?), - ( - "old_names", - mapped( - &native_field(&plan, "old_name_view_entries")?, - old_name_json, - )?, - ), - ], - )?; - let metadata: Bound<'_, PyAny> = native_field(&plan, "metadata")?; - if metadata.is_truthy()? { - result.set_item("metadata", metadata)?; - } - let synced_copy: Bound<'_, PyAny> = arguments.value(SYNCED_COPY_KEY)?; - if !synced_copy.is_none() { - result.set_item(SYNCED_COPY_KEY, synced_copy)?; - } - if arguments.passed(CHANGED_KEY)? { - result.set_item(CHANGED_KEY, arguments.value(CHANGED_KEY)?)?; - } - indented_json(result.as_any()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(format_plan_json, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs rename to crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index d950e7e477..9774b889a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -1,10 +1,11 @@ //! The text plan: planner objects read by attribute into the native plan report. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::plan_output::main::cursor_bound_display::cursor_bound_display; +use crate::plan_output::main::inclusive_cursor_end::inclusive_cursor_end; +use crate::plan_output::main::items::items; +use crate::plan_output::main::text::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::cli_text::main::format_plan_completion::format_plan_completion; -use sqlbuild_core::cli_text::main::format_plan_text::format_plan_text; use sqlbuild_core::cli_text::models::{ CliStyle, CursorPlanText, DisplayLimit, FunctionPlanText, ModelPlanText, PlanTextReport, PlanWarningText, ProviderUsageText, PythonPlanText, RenameHintText, SchemaFindingText, @@ -12,19 +13,18 @@ use sqlbuild_core::cli_text::models::{ }; use sqlbuild_core::plan_display::main::format_query_diff::format_query_diff; -use crate::bindings::_helpers::plan_output::cursor_details::{ +use crate::plan_output::_helpers::cursor_details::{ CursorDetails, DEFERRED, RESOLVED, cursor_details, runtime_inputs_clause, }; -use crate::bindings::_helpers::plan_output::display::{ - cursor_bound_display, format_no_input_rows, format_waiting_on_empty_inputs, - inclusive_cursor_end, model_materialization_label, +use crate::plan_output::_helpers::display::{ + format_no_input_rows, format_waiting_on_empty_inputs, model_materialization_label, }; -use crate::bindings::_helpers::plan_output::migrations::pre_build_lines; -use crate::bindings::_helpers::plan_output::plan_json::unified_diff; -use crate::bindings::_helpers::plan_output::python_identity::{ +use crate::plan_output::_helpers::migrations::pre_build_lines; +use crate::plan_output::_helpers::plan_json::unified_diff; +use crate::plan_output::_helpers::python_identity::{ definition_source_text, dependency_source_text, }; -use crate::bindings::_helpers::plan_output::values::{items, optional_text, text, texts}; +use crate::plan_output::_helpers::values::{optional_text, texts}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -37,7 +37,7 @@ const QUERY_CHANGED: &str = "query_changed"; const CONFIG_CHANGED: &str = "config_changed"; const MICROBATCH_MODE: &str = "microbatch"; const CHANGED_IDENTITY: &str = "changed"; -const PLAN_OPTIONS: (&[&str], &[&str]) = ( +pub(crate) const PLAN_OPTIONS: (&[&str], &[&str]) = ( &["plan"], &[ "full_refresh", @@ -439,7 +439,7 @@ pub(crate) fn display_limit(options: &Bound<'_, PyAny>) -> PyResult) -> PyResult { +pub(crate) fn report(arguments: &KeywordArguments<'_>) -> PyResult { let py: Python<'_> = arguments.value("plan")?.py(); let plan: Bound<'_, PyAny> = arguments.value("plan")?; let display_options: Bound<'_, PyAny> = { @@ -517,25 +517,3 @@ fn report(arguments: &KeywordArguments<'_>) -> PyResult { pre_build_lines: pre_build, }) } - -/// The text plan grouped by reason with inline detail, optionally closed by its completion line. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn format_plan(py: Python<'_>, options: Option>) -> PyResult { - let arguments: KeywordArguments<'_> = - KeywordArguments::read(py, "format_plan", options, PLAN_OPTIONS)?; - let report: PlanTextReport = report(&arguments)?; - let mut text: String = format_plan_text(&report); - if arguments.passed("changed_sections")? { - let sections: String = arguments.value("changed_sections")?.extract()?; - text = format!("{text}\n\n{sections}"); - } - if !arguments.flag("include_completion")? { - return Ok(text); - } - Ok(format!("{text}\n\n{}", format_plan_completion(&report))) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(format_plan, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/python_identity.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs rename to crates/sqlbuild-python-planner/src/plan_output/_helpers/python_identity.rs diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/safety.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/safety.rs new file mode 100644 index 0000000000..8f9e3e8d4f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/safety.rs @@ -0,0 +1,45 @@ +//! Structured future-cursor and maximum-start safety evidence for plan and result JSON. + +use crate::plan_output::main::dict::dict; +use crate::plan_output::main::maximum_start_safety::maximum_start_safety; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyString; + +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; + +/// `render(value=value)`. +pub(crate) fn render<'py>(value: Bound<'py, PyAny>) -> PyResult> { + Ok(PyString::new(value.py(), &render_value(&value)?).into_any()) +} + +/// `render(value=value)` when set, else `None`. +pub(crate) fn optional_render<'py>(value: Bound<'py, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(value); + } + render(value) +} + +pub(crate) fn bounds<'py>( + evidence: &Bound<'py, PyAny>, + (start, end): (&str, &str), +) -> PyResult> { + Ok(dict( + evidence.py(), + vec![ + ("start", render(native_field(evidence, start)?)?), + ("end", render(native_field(evidence, end)?)?), + ], + )? + .into_any()) +} + +/// Serialize maximum automatic-start safety evidence; `None` stays `None`. +#[pyfunction] +pub(crate) fn serialize_maximum_start_safety<'py>( + evidence: Bound<'py, PyAny>, +) -> PyResult> { + maximum_start_safety(&evidence) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs similarity index 69% rename from crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs rename to crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs index 61a5ab267c..1d5fe2e250 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs @@ -1,15 +1,11 @@ //! Reading planner values by attribute and building the JSON documents' dictionaries. -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList}; +use crate::plan_output::main::items::items; +use crate::plan_output::main::text::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyList; use sqlbuild_python_base::records::main::fields::field::field as native_field; -/// `str(value)`. -pub(crate) fn text(value: &Bound<'_, PyAny>) -> PyResult { - Ok(value.str()?.to_string()) -} - /// `str(value)`, or `None`. pub(crate) fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { if value.is_none() { @@ -23,11 +19,6 @@ pub(crate) fn texts(value: &Bound<'_, PyAny>) -> PyResult> { value.try_iter()?.map(|item| text(&item?)).collect() } -/// Every item of an iterable. -pub(crate) fn items<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { - value.try_iter()?.collect() -} - /// `value..value`, the value of an enum attribute. pub(crate) fn enum_value<'py>( value: &Bound<'py, PyAny>, @@ -44,26 +35,6 @@ pub(crate) fn optional_enum_value<'py>(value: &Bound<'py, PyAny>) -> PyResult( - py: Python<'py>, - entries: Vec<(&str, Bound<'py, PyAny>)>, -) -> PyResult> { - let built: Bound<'py, PyDict> = PyDict::new(py); - for (key, value) in entries { - built.set_item(key, value)?; - } - Ok(built) -} - -/// Any Rust value as a Python object. -pub(crate) fn any<'py, T: IntoPyObjectExt<'py>>( - py: Python<'py>, - value: T, -) -> PyResult> { - value.into_bound_py_any(py) -} - /// `list(value)`. pub(crate) fn list<'py>(value: &Bound<'py, PyAny>) -> PyResult> { Ok(PyList::new(value.py(), items(value)?)?.into_any()) diff --git a/crates/sqlbuild-python-planner/src/plan_output/constants.rs b/crates/sqlbuild-python-planner/src/plan_output/constants.rs new file mode 100644 index 0000000000..058f1c37cd --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/constants.rs @@ -0,0 +1,4 @@ +//! Public constants of plan output. + +/// The plan JSON key of what `--changed` would run, hold back and skip. +pub const CHANGED_KEY: &str = "changed"; diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/aligned_timestamp_bounds.rs b/crates/sqlbuild-python-planner/src/plan_output/main/aligned_timestamp_bounds.rs new file mode 100644 index 0000000000..f6d4c0437a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/aligned_timestamp_bounds.rs @@ -0,0 +1,25 @@ +//! Comparable bounds, presuming a naive side UTC when the other is aware. + +use crate::plan_output::_helpers::display::as_datetime; +use crate::plan_output::_helpers::display::presume_utc; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Comparable bounds, presuming a naive side UTC when the other is aware. +#[pyfunction] +#[pyo3(signature = (*, start, end))] +pub fn aligned_timestamp_bounds<'py>( + start: &Bound<'py, PyAny>, + end: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = start.py(); + let mut start_at: Bound<'py, PyAny> = as_datetime(start)?; + let mut end_at: Bound<'py, PyAny> = as_datetime(end)?; + if native_field(&start_at, "tzinfo")?.is_none() != native_field(&end_at, "tzinfo")?.is_none() { + start_at = presume_utc(start_at)?; + end_at = presume_utc(end_at)?; + } + PyTuple::new(py, [start_at, end_at]) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/any.rs b/crates/sqlbuild-python-planner/src/plan_output/main/any.rs new file mode 100644 index 0000000000..9788fce459 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/any.rs @@ -0,0 +1,9 @@ +//! Any Rust value as a Python object. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; + +/// Any Rust value as a Python object. +pub fn any<'py, T: IntoPyObjectExt<'py>>(py: Python<'py>, value: T) -> PyResult> { + value.into_bound_py_any(py) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/cursor_bound_display.rs b/crates/sqlbuild-python-planner/src/plan_output/main/cursor_bound_display.rs new file mode 100644 index 0000000000..efcca9a045 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/cursor_bound_display.rs @@ -0,0 +1,34 @@ +//! A cursor bound for display, collapsing a whole-day midnight timestamp to a date. + +use crate::plan_output::_helpers::display::parsed_bound; +use crate::plan_output::_helpers::values::optional_text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_core::plan_display::constants::INTEGER_CURSOR; +use sqlbuild_core::plan_display::main::cursor_bound_display::cursor_bound_display as bound_display; +use sqlbuild_core::plan_display::models::ParsedBound; + +/// A cursor bound for display, collapsing a whole-day midnight timestamp to a date. +#[pyfunction] +#[pyo3(signature = (*, value, cursor_type, cursor_grain))] +pub fn cursor_bound_display( + value: &Bound<'_, PyAny>, + cursor_type: &Bound<'_, PyAny>, + cursor_grain: &Bound<'_, PyAny>, +) -> PyResult { + let cursor_type: Option = optional_text(cursor_type)?; + let raw: String = value.extract()?; + let parsed: Option = if cursor_type.as_deref() == Some(INTEGER_CURSOR) { + None + } else { + parsed_bound(value)? + }; + Ok(bound_display( + &raw, + ( + cursor_type.as_deref(), + optional_text(cursor_grain)?.as_deref(), + ), + parsed.as_ref(), + )) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/dict.rs b/crates/sqlbuild-python-planner/src/plan_output/main/dict.rs new file mode 100644 index 0000000000..a75df8c775 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/dict.rs @@ -0,0 +1,16 @@ +//! A dictionary of the entries, in order. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; + +/// A dictionary of the entries, in order. +pub fn dict<'py>( + py: Python<'py>, + entries: Vec<(&str, Bound<'py, PyAny>)>, +) -> PyResult> { + let built: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in entries { + built.set_item(key, value)?; + } + Ok(built) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/format_plan.rs b/crates/sqlbuild-python-planner/src/plan_output/main/format_plan.rs new file mode 100644 index 0000000000..f2f4162f4a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/format_plan.rs @@ -0,0 +1,29 @@ +//! The text plan grouped by reason with inline detail, optionally closed by its completion line. + +use crate::plan_output::_helpers::plan_text::PLAN_OPTIONS; +use crate::plan_output::_helpers::plan_text::report; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_core::cli_text::main::format_plan_completion::format_plan_completion; +use sqlbuild_core::cli_text::main::format_plan_text::format_plan_text; +use sqlbuild_core::cli_text::models::PlanTextReport; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// The text plan grouped by reason with inline detail, optionally closed by its completion line. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn format_plan(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "format_plan", options, PLAN_OPTIONS)?; + let report: PlanTextReport = report(&arguments)?; + let mut text: String = format_plan_text(&report); + if arguments.passed("changed_sections")? { + let sections: String = arguments.value("changed_sections")?.extract()?; + text = format!("{text}\n\n{sections}"); + } + if !arguments.flag("include_completion")? { + return Ok(text); + } + Ok(format!("{text}\n\n{}", format_plan_completion(&report))) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/format_plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/main/format_plan_json.rs new file mode 100644 index 0000000000..c576d8b3a3 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/format_plan_json.rs @@ -0,0 +1,119 @@ +//! The plan as one JSON document, `json.dumps(indent=2)`. + +use crate::plan_output::_helpers::plan_json::column_migrations_json; +use crate::plan_output::_helpers::plan_json::function_json; +use crate::plan_output::_helpers::plan_json::migrations_json; +use crate::plan_output::_helpers::plan_json::model_json; +use crate::plan_output::_helpers::plan_json::old_name_json; +use crate::plan_output::_helpers::plan_json::providers_json; +use crate::plan_output::_helpers::plan_json::python_entry_json; +use crate::plan_output::_helpers::plan_json::retention_json; +use crate::plan_output::_helpers::plan_json::seed_json; +use crate::plan_output::_helpers::plan_json::source_load_json; +use crate::plan_output::_helpers::plan_json::table_type_json; +use crate::plan_output::_helpers::plan_json::warning_json; +use crate::plan_output::_helpers::values::mapped; +use crate::plan_output::constants::CHANGED_KEY; +use crate::plan_output::main::any::any; +use crate::plan_output::main::dict::dict; +use crate::plan_output::main::items::items; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyList}; +use sqlbuild_core::plan_display::constants::SYNCED_COPY_KEY; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::compile_outputs::main::indented_json::indented_json; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The plan as one JSON document, `json.dumps(indent=2)`. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn format_plan_json(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "format_plan_json", + options, + ( + &["plan"], + &["python_plan_entries", SYNCED_COPY_KEY, CHANGED_KEY], + ), + )?; + let plan: Bound<'_, PyAny> = arguments.value("plan")?; + let python_entries: Vec> = if arguments.passed("python_plan_entries")? { + items(&arguments.value("python_plan_entries")?)? + } else { + Vec::new() + }; + let length = |field: &str| -> PyResult { native_field(&plan, field)?.len() }; + let python_nodes: Vec> = python_entries + .iter() + .map(python_entry_json) + .collect::>()?; + let result: Bound<'_, PyDict> = dict( + py, + vec![ + ( + "selected_count", + any( + py, + length("model_entries")? + + length("seed_entries")? + + length("function_entries")?, + )?, + ), + ( + "source_load_count", + any(py, length("source_load_entries")?)?, + ), + ("python_node_count", any(py, python_nodes.len())?), + ( + "models", + mapped(&native_field(&plan, "model_entries")?, model_json)?, + ), + ( + "seeds", + mapped(&native_field(&plan, "seed_entries")?, seed_json)?, + ), + ( + "source_loads", + mapped( + &native_field(&plan, "source_load_entries")?, + source_load_json, + )?, + ), + ( + "functions", + mapped(&native_field(&plan, "function_entries")?, function_json)?, + ), + ("python_nodes", PyList::new(py, python_nodes)?.into_any()), + ("providers", providers_json(&plan, &python_entries)?), + ( + "warnings", + mapped(&native_field(&plan, "warnings")?, warning_json)?, + ), + ("retention", retention_json(&plan)?), + ("table_type_conversions", table_type_json(&plan)?), + ("migrations", migrations_json(&plan)?), + ("column_migrations", column_migrations_json(&plan)?), + ( + "old_names", + mapped( + &native_field(&plan, "old_name_view_entries")?, + old_name_json, + )?, + ), + ], + )?; + let metadata: Bound<'_, PyAny> = native_field(&plan, "metadata")?; + if metadata.is_truthy()? { + result.set_item("metadata", metadata)?; + } + let synced_copy: Bound<'_, PyAny> = arguments.value(SYNCED_COPY_KEY)?; + if !synced_copy.is_none() { + result.set_item(SYNCED_COPY_KEY, synced_copy)?; + } + if arguments.passed(CHANGED_KEY)? { + result.set_item(CHANGED_KEY, arguments.value(CHANGED_KEY)?)?; + } + indented_json(result.as_any()) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/future_cursor_safety.rs b/crates/sqlbuild-python-planner/src/plan_output/main/future_cursor_safety.rs new file mode 100644 index 0000000000..f890726eae --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/future_cursor_safety.rs @@ -0,0 +1,81 @@ +//! Cap policy, bounds and physical input evidence of future-cursor safety, or `None`. + +use crate::plan_output::_helpers::safety::bounds; +use crate::plan_output::_helpers::safety::optional_render; +use crate::plan_output::_helpers::safety::render; +use crate::plan_output::_helpers::values::enum_value; +use crate::plan_output::main::dict::dict; +use crate::plan_output::main::items::items; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Cap policy, bounds and physical input evidence of future-cursor safety, or `None`. +pub fn future_cursor_safety<'py>(evidence: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = evidence.py(); + if evidence.is_none() { + return Ok(evidence.clone()); + } + let relation: Bound<'py, PyAny> = native_field(evidence, "determining_relation")?; + let determining: Bound<'py, PyAny> = if relation.is_none() { + relation + } else { + dict( + py, + vec![ + ("relation", relation), + ( + "cursor_column", + native_field(evidence, "determining_cursor_column")?, + ), + ], + )? + .into_any() + }; + let mut inputs: Vec> = Vec::new(); + for item in items(&native_field(evidence, "inputs")?)? { + inputs.push(dict( + py, + vec![ + ("relation", native_field(&item, "relation")?), + ("cursor_column", native_field(&item, "cursor_column")?), + ("minimum", optional_render(native_field(&item, "minimum")?)?), + ("maximum", render(native_field(&item, "maximum")?)?), + ], + )?); + } + Ok(dict( + py, + vec![ + ("action", enum_value(evidence, "action")?), + ("max_distance", native_field(evidence, "max_distance")?), + ( + "invocation_time", + render(native_field(evidence, "invocation_time")?)?, + ), + ( + "discovered_bounds", + bounds(evidence, ("discovered_start", "discovered_end"))?, + ), + ( + "applied_bounds", + bounds(evidence, ("applied_start", "applied_end"))?, + ), + ( + "maximum_allowed_bounds", + bounds(evidence, ("maximum_allowed_start", "maximum_allowed_end"))?, + ), + ( + "future_start_detected", + native_field(evidence, "future_start_detected")?, + ), + ( + "future_end_detected", + native_field(evidence, "future_end_detected")?, + ), + ("determining_input", determining), + ("inputs", pyo3::types::PyList::new(py, inputs)?.into_any()), + ], + )? + .into_any()) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/inclusive_cursor_end.rs b/crates/sqlbuild-python-planner/src/plan_output/main/inclusive_cursor_end.rs new file mode 100644 index 0000000000..416956edfd --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/inclusive_cursor_end.rs @@ -0,0 +1,23 @@ +//! The last cursor value included by an exclusive end bound. + +use crate::plan_output::_helpers::display::cursor_typing; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; +use sqlbuild_python_front::cursor::main::scalars::to_inclusive::to_inclusive; +use sqlbuild_python_front::cursor::main::scalars::try_parse::try_parse; + +/// The last cursor value included by an exclusive end bound. +#[pyfunction] +#[pyo3(signature = (*, end, cursor_type, cursor_grain))] +pub fn inclusive_cursor_end( + end: &Bound<'_, PyAny>, + cursor_type: &Bound<'_, PyAny>, + cursor_grain: &Bound<'_, PyAny>, +) -> PyResult { + let (effective, grain) = cursor_typing(cursor_type, cursor_grain)?; + let Some(parsed) = try_parse(end, &effective)? else { + return end.extract(); + }; + render_value(&to_inclusive(&parsed, &grain)?) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/items.rs b/crates/sqlbuild-python-planner/src/plan_output/main/items.rs new file mode 100644 index 0000000000..beb0d4d41b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/items.rs @@ -0,0 +1,8 @@ +//! Every item of an iterable. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +/// Every item of an iterable. +pub fn items<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { + value.try_iter()?.collect() +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/maximum_start_safety.rs b/crates/sqlbuild-python-planner/src/plan_output/main/maximum_start_safety.rs new file mode 100644 index 0000000000..2144ec8b1c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/maximum_start_safety.rs @@ -0,0 +1,55 @@ +//! Automatic-start policy and physical eligibility evidence, or `None`. + +use crate::plan_output::_helpers::safety::optional_render; +use crate::plan_output::_helpers::safety::render; +use crate::plan_output::_helpers::values::enum_value; +use crate::plan_output::main::dict::dict; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Automatic-start policy and physical eligibility evidence, or `None`. +pub fn maximum_start_safety<'py>(evidence: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = evidence.py(); + if evidence.is_none() { + return Ok(evidence.clone()); + } + Ok(dict( + py, + vec![ + ("action", enum_value(evidence, "action")?), + ("max_ahead", native_field(evidence, "max_ahead")?), + ( + "invocation_time", + render(native_field(evidence, "invocation_time")?)?, + ), + ( + "physical_target_max", + render(native_field(evidence, "physical_target_max")?)?, + ), + ( + "highest_eligible_target_max", + optional_render(native_field(evidence, "highest_eligible_target_max")?)?, + ), + ( + "effective_start", + render(native_field(evidence, "effective_start")?)?, + ), + ( + "maximum_allowed_start", + render(native_field(evidence, "maximum_allowed_start")?)?, + ), + ( + "input", + dict( + py, + vec![ + ("relation", native_field(evidence, "target_relation")?), + ("cursor_column", native_field(evidence, "cursor_column")?), + ], + )? + .into_any(), + ), + ], + )? + .into_any()) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/mod.rs b/crates/sqlbuild-python-planner/src/plan_output/main/mod.rs new file mode 100644 index 0000000000..1ffbc27d2f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/mod.rs @@ -0,0 +1,19 @@ +pub mod aligned_timestamp_bounds; +pub mod any; +pub mod cursor_bound_display; +pub mod dict; +pub mod format_plan; +pub mod format_plan_json; +pub mod future_cursor_safety; +pub mod inclusive_cursor_end; +pub mod items; +pub mod maximum_start_safety; +pub mod optional_facts; +pub mod register_display; +pub mod register_migrations; +pub mod register_plan_json; +pub mod register_plan_text; +pub mod register_safety; +pub mod resolve_bounded_cursor_override; +pub mod resolve_effective_microbatch_batch_size; +pub mod text; diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/optional_facts.rs b/crates/sqlbuild-python-planner/src/plan_output/main/optional_facts.rs new file mode 100644 index 0000000000..2439cc7e29 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/optional_facts.rs @@ -0,0 +1,13 @@ +//! Optional facts. + +use crate::plan_output::_helpers::display::label_facts; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::plan_display::models::ModelLabelFacts; + +/// Optional facts. +pub fn optional_facts(entry: &Bound<'_, PyAny>) -> PyResult> { + if entry.is_none() { + return Ok(None); + } + label_facts(entry).map(Some) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs b/crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs new file mode 100644 index 0000000000..63f14dcf35 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs @@ -0,0 +1,22 @@ +//! Register the display functions and classes. + +use crate::plan_output::_helpers::display::format_waiting_on_empty_inputs; +use crate::plan_output::main::aligned_timestamp_bounds::aligned_timestamp_bounds; +use crate::plan_output::main::cursor_bound_display::cursor_bound_display; +use crate::plan_output::main::inclusive_cursor_end::inclusive_cursor_end; +use crate::plan_output::main::resolve_effective_microbatch_batch_size::resolve_effective_microbatch_batch_size; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the display functions and classes. +pub fn register_display(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(inclusive_cursor_end, module)?)?; + module.add_function(wrap_pyfunction!(cursor_bound_display, module)?)?; + module.add_function(wrap_pyfunction!(format_waiting_on_empty_inputs, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_effective_microbatch_batch_size, + module + )?)?; + module.add_function(wrap_pyfunction!(aligned_timestamp_bounds, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/register_migrations.rs b/crates/sqlbuild-python-planner/src/plan_output/main/register_migrations.rs new file mode 100644 index 0000000000..8004dd6c69 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/register_migrations.rs @@ -0,0 +1,12 @@ +//! Register the migrations functions and classes. + +use crate::plan_output::_helpers::migrations::format_pre_build; +use crate::plan_output::_helpers::migrations::render_old_name; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the migrations functions and classes. +pub fn register_migrations(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_pre_build, module)?)?; + module.add_function(wrap_pyfunction!(render_old_name, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_json.rs new file mode 100644 index 0000000000..1d8c833f94 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_json.rs @@ -0,0 +1,10 @@ +//! Register the plan json functions and classes. + +use crate::plan_output::main::format_plan_json::format_plan_json; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the plan json functions and classes. +pub fn register_plan_json(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_plan_json, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_text.rs new file mode 100644 index 0000000000..52e691c6a4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/register_plan_text.rs @@ -0,0 +1,10 @@ +//! Register the plan text functions and classes. + +use crate::plan_output::main::format_plan::format_plan; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the plan text functions and classes. +pub fn register_plan_text(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_plan, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/register_safety.rs b/crates/sqlbuild-python-planner/src/plan_output/main/register_safety.rs new file mode 100644 index 0000000000..b9fe38126e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/register_safety.rs @@ -0,0 +1,10 @@ +//! Register the safety functions and classes. + +use crate::plan_output::_helpers::safety::serialize_maximum_start_safety; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the safety functions and classes. +pub fn register_safety(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(serialize_maximum_start_safety, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/resolve_bounded_cursor_override.rs b/crates/sqlbuild-python-planner/src/plan_output/main/resolve_bounded_cursor_override.rs new file mode 100644 index 0000000000..4dcc366b3e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/resolve_bounded_cursor_override.rs @@ -0,0 +1,34 @@ +//! A fully bounded inclusive operator override as an executable interval. + +use crate::plan_output::_helpers::display::cursor_typing; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cursor::main::scalars::to_exclusive::to_exclusive; +use sqlbuild_python_front::cursor::main::scalars::try_parse::try_parse; + +/// A fully bounded inclusive operator override as an executable interval. +pub fn resolve_bounded_cursor_override<'py>( + start_cursor_override: &Bound<'py, PyAny>, + end_cursor_override: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, + cursor_grain: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = start_cursor_override.py(); + if start_cursor_override.is_none() || end_cursor_override.is_none() { + return Ok(None); + } + let (effective, grain) = cursor_typing(cursor_type, cursor_grain)?; + let (Some(start), Some(end)) = ( + try_parse(start_cursor_override, &effective)?, + try_parse(end_cursor_override, &effective)?, + ) else { + return Ok(None); + }; + new_record_with( + py, + RecordId::CursorBounds, + &[("start", start), ("end", to_exclusive(&end, &grain)?)], + ) + .map(Some) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/resolve_effective_microbatch_batch_size.rs b/crates/sqlbuild-python-planner/src/plan_output/main/resolve_effective_microbatch_batch_size.rs new file mode 100644 index 0000000000..612c04153b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/resolve_effective_microbatch_batch_size.rs @@ -0,0 +1,24 @@ +//! The explicit effective batch-size token resolved against the replay grain. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{PyAnyMethods, PyErr, Python}; +use pyo3::pyfunction; +use sqlbuild_core::plan_display::main::effective_microbatch_batch_size::effective_microbatch_batch_size; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::types::RecordId; + +/// The explicit effective batch-size token resolved against the replay grain. +#[pyfunction] +#[pyo3(signature = (*, batch_size, effective_grain))] +pub fn resolve_effective_microbatch_batch_size( + py: Python<'_>, + batch_size: &str, + effective_grain: &str, +) -> Result { + effective_microbatch_batch_size(batch_size, effective_grain).map_err(|error| { + record_class(py, RecordId::CursorGrain) + .and_then(|grain| grain.call1((error.0,))) + .err() + .unwrap_or_else(|| PyValueError::new_err("unknown cursor grain")) + }) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/text.rs b/crates/sqlbuild-python-planner/src/plan_output/main/text.rs new file mode 100644 index 0000000000..e92c227b48 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/text.rs @@ -0,0 +1,8 @@ +//! `str(value)`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +/// `str(value)`. +pub fn text(value: &Bound<'_, PyAny>) -> PyResult { + Ok(value.str()?.to_string()) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/mod.rs b/crates/sqlbuild-python-planner/src/plan_output/mod.rs new file mode 100644 index 0000000000..2560516c60 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/mod.rs @@ -0,0 +1,5 @@ +//! Plan output: the text and JSON plan documents, display fragments, cursor details and safety notes. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; diff --git a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/actions.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/actions.rs new file mode 100644 index 0000000000..c8a3efa9b8 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/actions.rs @@ -0,0 +1,4 @@ +//! Per-model actions from each model's own change, and the incremental replacement predicate. + +pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +pub(crate) const CREATE_TABLE_ACTION: &str = "create_table"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs rename to crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs index f5a3ad7932..d22a1fa85d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs @@ -1,12 +1,14 @@ //! Detect how each selected model and function changed since its recorded build. +use crate::planner_changes::main::native_enum::native_enum; +use crate::planner_changes::main::planner_enum::planner_enum; +use crate::planner_changes::models::ModelPass; use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::planner_changes::results::native_enum; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_front::contracts::main::matching_dynamic_families::matching_dynamic_families; use sqlbuild_python_front::model_config::main::config::config_text::config_text; @@ -31,14 +33,14 @@ use sqlbuild_analysis::planner_changes::models::{ SchemaFindingRow, }; -use crate::bindings::_helpers::planner_changes::renames::origin_names; -use crate::bindings::_helpers::planner_changes::results::{ - backfill_result, change_kind, planner_enum, replay_policy_error, schema_findings, -}; -use crate::bindings::_helpers::planner_identity::metadata::{ - compiled_function_fingerprint_sql, function_local_hashes, model_declared_columns_hash, - model_version_identity_metadata_json, +use crate::planner_changes::_helpers::renames::origin_names; +use crate::planner_changes::_helpers::results::{ + backfill_result, change_kind, replay_policy_error, schema_findings, }; +use crate::planner_identity::main::compiled_function_fingerprint_sql::compiled_function_fingerprint_sql; +use crate::planner_identity::main::function_local_hashes::function_local_hashes; +use crate::planner_identity::main::model_declared_columns_hash::model_declared_columns_hash; +use crate::planner_identity::main::model_version_identity_metadata_json::model_version_identity_metadata_json; use sqlbuild_project::types::InferredRow; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -60,19 +62,10 @@ use sqlbuild_python_project::semantic_shapes::main::snapshot_validity_columns::s const CHANGES_LOGGER: &str = "sqlbuild.planner.changes"; const REPLAY_CONFIG_KEY: &str = "replay_on_change"; -/// Inputs shared by every model of one detection pass. -pub(crate) struct ModelPass<'py> { - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) query_change_tracking: bool, - pub(crate) full_refresh: bool, - pub(crate) function_local_hashes: Bound<'py, PyAny>, - pub(crate) hook_version_hashes: Bound<'py, PyAny>, -} - /// Selected model and function changes. #[pyfunction(name = "detect_changes")] #[pyo3(signature = (**options))] -fn detect_changes_of<'py>( +pub(crate) fn detect_changes_of<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -170,50 +163,6 @@ pub(crate) fn detect_changes<'py>( ) } -/// One model's change and backfill. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn detect_model_changes<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "detect_model_changes", - options, - ( - &[ - "model", - "snapshot", - "sql_analysis_enabled", - "query_change_tracking", - "full_refresh", - ], - &[ - "function_local_hashes", - "hook_version_hashes", - "expected_version_hash", - "expected_metadata_json", - ], - ), - )?; - let pass = ModelPass { - snapshot: arguments.value("snapshot")?, - query_change_tracking: arguments.flag("query_change_tracking")?, - full_refresh: arguments.flag("full_refresh")?, - function_local_hashes: arguments.value("function_local_hashes")?, - hook_version_hashes: arguments.value("hook_version_hashes")?, - }; - model_change( - &pass, - &arguments.value("model")?, - ( - arguments.value("expected_version_hash")?.extract()?, - arguments.value("expected_metadata_json")?.extract()?, - ), - ) -} - pub(crate) fn scope_model_changes<'py>( (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), query_change_tracking: Option, @@ -599,7 +548,7 @@ fn schema_findings_of<'py>( /// Schema findings between declared or inferred columns and warehouse columns. #[pyfunction(name = "detect_schema_changes")] #[pyo3(signature = (**options))] -fn schema_changes<'py>( +pub(crate) fn schema_changes<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -766,9 +715,3 @@ impl<'py> ChangeResult<'py> { ) } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(detect_changes_of, module)?)?; - module.add_function(wrap_pyfunction!(detect_model_changes, module)?)?; - module.add_function(wrap_pyfunction!(schema_changes, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs similarity index 56% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs rename to crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs index 6946656b3c..cba6513864 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs @@ -1,11 +1,10 @@ //! Version-identity metadata helpers the remaining Python planner reads. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::planner_changes::main::version_identity_payload::version_identity_payload; use sqlbuild_analysis::planner_changes::main::with_declared_columns_hash::with_declared_columns_hash as with_hash; -use sqlbuild_analysis::planner_changes::main::without_declared_columns_hash::without_declared_columns_hash as without_hash; use sqlbuild_analysis::planner_changes::models::IdentityPayload; use sqlbuild_python_base::boundary::main::imported::imported; @@ -13,7 +12,7 @@ use sqlbuild_python_base::boundary::main::imported::imported; /// The decoded metadata that participates in version identity, the raw text when it is not JSON, or `None`. #[pyfunction] #[pyo3(signature = (metadata_json))] -fn version_identity_metadata_payload<'py>( +pub(crate) fn version_identity_metadata_payload<'py>( py: Python<'py>, metadata_json: Option<&str>, ) -> PyResult> { @@ -29,18 +28,9 @@ fn version_identity_metadata_payload<'py>( /// Fingerprint metadata JSON that also records the model's declared-columns hash. #[pyfunction] #[pyo3(signature = (*, metadata_json, declared_columns_hash))] -fn with_declared_columns_hash(metadata_json: &str, declared_columns_hash: &str) -> String { +pub(crate) fn with_declared_columns_hash( + metadata_json: &str, + declared_columns_hash: &str, +) -> String { with_hash(metadata_json, declared_columns_hash) } - -/// Fingerprint metadata JSON without the declared-columns hash key, if present. -#[pyfunction] -pub(crate) fn without_declared_columns_hash(metadata_json: &str) -> String { - without_hash(metadata_json) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(version_identity_metadata_payload, module)?)?; - module.add_function(wrap_pyfunction!(with_declared_columns_hash, module)?)?; - module.add_function(wrap_pyfunction!(without_declared_columns_hash, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/mod.rs new file mode 100644 index 0000000000..2f2b6ad173 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod actions; +pub(crate) mod detect; +pub(crate) mod metadata; +pub(crate) mod renames; +pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs rename to crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs index 37a0ab4c90..dba4aa0cce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs @@ -5,9 +5,9 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_project::fingerprints::main::compute_query_hash::compute_query_hash; use std::collections::BTreeSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_changes::main::origin_reference_names::origin_reference_names; use sqlbuild_analysis::planner_changes::models::{NameSpan, RenameCandidate}; @@ -123,7 +123,7 @@ fn utc<'py>(value: &Bound<'py, PyAny>) -> PyResult> { /// The rename mapping under which `query_sql` hashes to the recorded definition, or `None`. #[pyfunction(name = "origin_reference_names")] #[pyo3(signature = (*, query_sql, recorded, renames, dialect))] -fn origin_reference_names_of<'py>( +pub(crate) fn origin_reference_names_of<'py>( query_sql: String, recorded: &Bound<'py, PyAny>, renames: &Bound<'py, PyAny>, @@ -262,7 +262,3 @@ fn changed_referencing_query( } Ok(false) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(origin_reference_names_of, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/results.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs rename to crates/sqlbuild-python-planner/src/planner_changes/_helpers/results.rs index 1e9fe96a0f..6a5d138a2c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/results.rs @@ -1,5 +1,7 @@ //! The planner's Python change-detection result models. +use crate::planner_changes::main::native_enum::native_enum; +use crate::planner_changes::main::planner_enum::planner_enum; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyTuple; use pyo3::{IntoPyObject, PyErr}; @@ -10,7 +12,6 @@ use sqlbuild_analysis::planner_changes::models::{ Backfill, BackfillAction, ChangeKind, SchemaChangeKind, SchemaColumnSource, SchemaFindingRow, }; -use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; @@ -20,24 +21,6 @@ pub(crate) fn replay_policy_error(py: Python<'_>, error: UnknownReplayPolicy) -> new_plain_error(py, ErrorId::PlannerInput, &error.message()) } -/// `TYPES_MODULE.(value)`. -pub(crate) fn planner_enum<'py>( - py: Python<'py>, - name: RecordId, - value: &str, -) -> PyResult> { - new_record(py, name, (value,), None) -} - -/// A member of a natively declared planner enumeration, from its value. -pub(crate) fn native_enum<'py>( - py: Python<'py>, - name: RecordId, - value: &str, -) -> PyResult> { - new_record(py, name, (value,), None) -} - /// `BackfillResult(action=..., duration=...)`. pub(crate) fn backfill_result<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs new file mode 100644 index 0000000000..fee967c759 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs @@ -0,0 +1,14 @@ +//! Selected model and function changes against expected version hashes and metadata JSON. + +use crate::planner_changes::_helpers::detect; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Selected model and function changes against expected version hashes and metadata JSON. +#[inline] +pub fn detect_changes<'py>( + (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + full_refresh: bool, + expected: (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + detect::detect_changes((project, scope, snapshot), full_refresh, expected) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/detect_model_changes.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_model_changes.rs new file mode 100644 index 0000000000..cf5a66951f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_model_changes.rs @@ -0,0 +1,52 @@ +//! One model's change and backfill. + +use crate::planner_changes::_helpers::detect::model_change; +use crate::planner_changes::models::ModelPass; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// One model's change and backfill. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn detect_model_changes<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "detect_model_changes", + options, + ( + &[ + "model", + "snapshot", + "sql_analysis_enabled", + "query_change_tracking", + "full_refresh", + ], + &[ + "function_local_hashes", + "hook_version_hashes", + "expected_version_hash", + "expected_metadata_json", + ], + ), + )?; + let pass = ModelPass { + snapshot: arguments.value("snapshot")?, + query_change_tracking: arguments.flag("query_change_tracking")?, + full_refresh: arguments.flag("full_refresh")?, + function_local_hashes: arguments.value("function_local_hashes")?, + hook_version_hashes: arguments.value("hook_version_hashes")?, + }; + model_change( + &pass, + &arguments.value("model")?, + ( + arguments.value("expected_version_hash")?.extract()?, + arguments.value("expected_metadata_json")?.extract()?, + ), + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/mod.rs new file mode 100644 index 0000000000..d6db35a20a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/mod.rs @@ -0,0 +1,14 @@ +pub mod detect_changes; +pub mod detect_model_changes; +pub mod model_change; +pub mod native_enum; +pub mod plan_reference_renames; +pub mod planner_enum; +pub mod register_actions; +pub mod register_detect; +pub mod register_metadata; +pub mod register_renames; +pub mod replaces_incremental_relation; +pub mod resolve_model_actions; +pub mod scope_model_changes; +pub mod without_declared_columns_hash; diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/model_change.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/model_change.rs new file mode 100644 index 0000000000..f3640bac39 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/model_change.rs @@ -0,0 +1,15 @@ +//! Model change. + +use crate::planner_changes::_helpers::detect; +use crate::planner_changes::models::ModelPass; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Model change. +#[inline] +pub fn model_change<'py>( + pass: &ModelPass<'py>, + model: &Bound<'py, PyAny>, + (expected_version_hash, expected_metadata_json): (Option, Option), +) -> PyResult> { + detect::model_change(pass, model, (expected_version_hash, expected_metadata_json)) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/native_enum.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/native_enum.rs new file mode 100644 index 0000000000..ed1872a423 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/native_enum.rs @@ -0,0 +1,14 @@ +//! A member of a natively declared planner enumeration, from its value. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; + +/// A member of a natively declared planner enumeration, from its value. +pub fn native_enum<'py>( + py: Python<'py>, + name: RecordId, + value: &str, +) -> PyResult> { + new_record(py, name, (value,), None) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/plan_reference_renames.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/plan_reference_renames.rs new file mode 100644 index 0000000000..27825bee37 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/plan_reference_renames.rs @@ -0,0 +1,14 @@ +//! This plan's renames plus recorded renames, read only when a selected query that references models changed. + +use crate::planner_changes::_helpers::renames; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; + +/// This plan's renames plus recorded renames, read only when a selected query that references models changed. +#[inline] +pub fn plan_reference_renames<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + renames::plan_reference_renames(py, options) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/planner_enum.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/planner_enum.rs new file mode 100644 index 0000000000..9f90be5077 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/planner_enum.rs @@ -0,0 +1,14 @@ +//! `TYPES_MODULE.(value)`. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; + +/// `TYPES_MODULE.(value)`. +pub fn planner_enum<'py>( + py: Python<'py>, + name: RecordId, + value: &str, +) -> PyResult> { + new_record(py, name, (value,), None) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/register_actions.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/register_actions.rs new file mode 100644 index 0000000000..185515e8cf --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/register_actions.rs @@ -0,0 +1,10 @@ +//! Register the actions functions and classes. + +use crate::planner_changes::main::replaces_incremental_relation::replaces_incremental_relation; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the actions functions and classes. +pub fn register_actions(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(replaces_incremental_relation, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs new file mode 100644 index 0000000000..9a13e5a439 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs @@ -0,0 +1,14 @@ +//! Register the detect functions and classes. + +use crate::planner_changes::_helpers::detect::detect_changes_of; +use crate::planner_changes::_helpers::detect::schema_changes; +use crate::planner_changes::main::detect_model_changes::detect_model_changes; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the detect functions and classes. +pub fn register_detect(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(detect_changes_of, module)?)?; + module.add_function(wrap_pyfunction!(detect_model_changes, module)?)?; + module.add_function(wrap_pyfunction!(schema_changes, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs new file mode 100644 index 0000000000..abacb50745 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs @@ -0,0 +1,14 @@ +//! Register the metadata functions and classes. + +use crate::planner_changes::_helpers::metadata::version_identity_metadata_payload; +use crate::planner_changes::_helpers::metadata::with_declared_columns_hash; +use crate::planner_changes::main::without_declared_columns_hash::without_declared_columns_hash; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the metadata functions and classes. +pub fn register_metadata(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(version_identity_metadata_payload, module)?)?; + module.add_function(wrap_pyfunction!(with_declared_columns_hash, module)?)?; + module.add_function(wrap_pyfunction!(without_declared_columns_hash, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/register_renames.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/register_renames.rs new file mode 100644 index 0000000000..d267b7a6b9 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/register_renames.rs @@ -0,0 +1,10 @@ +//! Register the renames functions and classes. + +use crate::planner_changes::_helpers::renames::origin_reference_names_of; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the renames functions and classes. +pub fn register_renames(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(origin_reference_names_of, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/replaces_incremental_relation.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/replaces_incremental_relation.rs new file mode 100644 index 0000000000..68df46256e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/replaces_incremental_relation.rs @@ -0,0 +1,12 @@ +//! Incremental replacements, whose bounds must ignore the destination maximum. + +use crate::planner_changes::_helpers::actions::CREATE_TABLE_ACTION; +use crate::planner_changes::_helpers::actions::INCREMENTAL_MATERIALIZATION; +use pyo3::pyfunction; + +/// Incremental replacements, whose bounds must ignore the destination maximum. +#[pyfunction] +#[pyo3(signature = (*, materialization_type, action))] +pub fn replaces_incremental_relation(materialization_type: &str, action: &str) -> bool { + materialization_type == INCREMENTAL_MATERIALIZATION && action == CREATE_TABLE_ACTION +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/resolve_model_actions.rs similarity index 65% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs rename to crates/sqlbuild-python-planner/src/planner_changes/main/resolve_model_actions.rs index 2b433681a5..ff5cfa8958 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/resolve_model_actions.rs @@ -1,21 +1,16 @@ -//! Per-model actions from each model's own change, and the incremental replacement predicate. +//! Each selected model's change with the backfill its own `replay_on_change` decided. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_scope::constants::MODEL; - use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; -const INCREMENTAL_MATERIALIZATION: &str = "incremental"; -const CREATE_TABLE_ACTION: &str = "create_table"; - /// Each selected model's change with the backfill its own `replay_on_change` decided. -pub(crate) fn resolve_model_actions<'py>( +pub fn resolve_model_actions<'py>( scope: &Bound<'py, PyAny>, changes: &Bound<'py, PyAny>, ) -> PyResult> { @@ -48,14 +43,3 @@ pub(crate) fn resolve_model_actions<'py>( &[("models", resolved.into_any())], ) } - -/// Incremental replacements, whose bounds must ignore the destination maximum. -#[pyfunction] -#[pyo3(signature = (*, materialization_type, action))] -pub(crate) fn replaces_incremental_relation(materialization_type: &str, action: &str) -> bool { - materialization_type == INCREMENTAL_MATERIALIZATION && action == CREATE_TABLE_ACTION -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(replaces_incremental_relation, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/scope_model_changes.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/scope_model_changes.rs new file mode 100644 index 0000000000..82e98e9045 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/scope_model_changes.rs @@ -0,0 +1,19 @@ +//! Scope model changes. + +use crate::planner_changes::_helpers::detect; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyDict; + +/// Scope model changes. +#[inline] +pub fn scope_model_changes<'py>( + (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + query_change_tracking: Option, + (version_hashes, metadata_jsons): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + detect::scope_model_changes( + (project, scope, snapshot), + query_change_tracking, + (version_hashes, metadata_jsons), + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/without_declared_columns_hash.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/without_declared_columns_hash.rs new file mode 100644 index 0000000000..4e60201ec8 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/without_declared_columns_hash.rs @@ -0,0 +1,10 @@ +//! Fingerprint metadata JSON without the declared-columns hash key, if present. + +use pyo3::pyfunction; +use sqlbuild_analysis::planner_changes::main::without_declared_columns_hash::without_declared_columns_hash as without_hash; + +/// Fingerprint metadata JSON without the declared-columns hash key, if present. +#[pyfunction] +pub fn without_declared_columns_hash(metadata_json: &str) -> String { + without_hash(metadata_json) +} diff --git a/crates/sqlbuild-python-planner/src/planner_changes/mod.rs b/crates/sqlbuild-python-planner/src/planner_changes/mod.rs new file mode 100644 index 0000000000..0c35d95139 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/mod.rs @@ -0,0 +1,5 @@ +//! Planner change detection: how each model changed, its actions, renames and identity metadata. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-planner/src/planner_changes/models.rs b/crates/sqlbuild-python-planner/src/planner_changes/models.rs new file mode 100644 index 0000000000..ccd18e88d1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_changes/models.rs @@ -0,0 +1,14 @@ +//! Public records of planner changes. + +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; + +/// Inputs shared by every model of one detection pass. +#[derive(Debug)] +pub struct ModelPass<'py> { + pub snapshot: Bound<'py, PyAny>, + pub query_change_tracking: bool, + pub full_refresh: bool, + pub function_local_hashes: Bound<'py, PyAny>, + pub hook_version_hashes: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/contexts.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/contexts.rs index f6b65deb03..601e006dbb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/contexts.rs @@ -1,15 +1,14 @@ //! Relation, model-plan and entry-build contexts: built from keyword arguments and read by plan entry construction. +use crate::planner_entries::main::entries::default_microbatch_limit_action::default_microbatch_limit_action; use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; use pyo3::types::{PyDict, PyFrozenSet}; -use crate::bindings::_helpers::planner_entries::models::{ +use crate::planner_entries::models::{ ModelPlanContext, PlanEntryBuildInputs, PlannerRelationsContext, }; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::types::RecordId; /// The argument `name`, or `default()` when it was not passed. fn value_or<'py>( @@ -121,7 +120,7 @@ impl ModelPlanContext { impl PlanEntryBuildInputs { #[new] #[pyo3(signature = (**kwargs))] - pub(crate) fn new(py: Python<'_>, kwargs: Option>) -> PyResult { + pub fn new(py: Python<'_>, kwargs: Option>) -> PyResult { let args = KeywordArguments::read( py, "PlanEntryBuildInputs", @@ -164,8 +163,3 @@ impl PlanEntryBuildInputs { }) } } - -/// The microbatch limit action used when a project configures none. -pub(crate) fn default_microbatch_limit_action(py: Python<'_>) -> PyResult> { - enum_member(py, RecordId::MicrobatchLimitAction, "ERROR") -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/getters.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/getters.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs index 11b359c3e3..c98bdf79f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/getters.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs @@ -3,29 +3,11 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::planner_entries::models::{ +use crate::planner_entries::models::{ ModelPlanContext, PlanEntryBuildInputs, PlannerRelationsContext, }; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[ - GetterEntry { - class: class_of::, - read: read_model_plan_context, - }, - GetterEntry { - class: class_of::, - read: read_plan_entry_build_inputs, - }, - GetterEntry { - class: class_of::, - read: read_planner_relations_context, - }, -]; - -fn read_model_plan_context<'py>( +pub(crate) fn read_model_plan_context<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -55,7 +37,7 @@ fn read_model_plan_context<'py>( }) } -fn read_plan_entry_build_inputs<'py>( +pub(crate) fn read_plan_entry_build_inputs<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -87,7 +69,7 @@ fn read_plan_entry_build_inputs<'py>( }) } -fn read_planner_relations_context<'py>( +pub(crate) fn read_planner_relations_context<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/mod.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/mod.rs new file mode 100644 index 0000000000..8147db06b4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod contexts; +pub(crate) mod getters; +pub(crate) mod model_entries; +pub(crate) mod relations; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs similarity index 81% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs index 91bedada89..49b574f80d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs @@ -1,43 +1,42 @@ //! Per-model plan entries from resolved change results and warehouse snapshot facts. -use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; -use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; -use crate::bindings::_helpers::planner_entries::models::{ - ModelPlanContext, PlanEntriesRequest, PlanEntryBuildInputs, PlannerRelationsContext, -}; -use crate::bindings::_helpers::planner_entries::values::{ +use crate::planner_changes::main::replaces_incremental_relation::replaces_incremental_relation; +use crate::planner_entries::_helpers::entries::values::{ MICROBATCH_CONCURRENCY_MESSAGE, error_of, native, none, planner_type, replaced, string, }; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::full_refresh::{ - check_full_rebuild_allowed, describe_full_rebuild_cause, resolve_model_full_refresh, -}; -use crate::bindings::_helpers::planner_phases::models::ModelEntryResults; -use crate::bindings::_helpers::planner_resolution::cursor_ranges::{ - CursorWindowInputs, config_str, cursor_safety_configs, empty_cursor_inputs, microbatch_range, - model_cursor_overrides, plan_cursor_bounds, -}; -use crate::bindings::_helpers::planner_resolution::cursor_warnings::with_cursor_safety_warnings; -use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest as SqlRequest; -use crate::bindings::_helpers::planner_resolution::orchestration::{ - Resolution, model_hook_entries, resolve_model_sql, -}; -use crate::bindings::_helpers::planner_strategy::cursor_inputs::runtime_cursor_producer_names; -use crate::bindings::_helpers::planner_strategy::cursor_inputs::{ - build_cursor_input_relations, reject_unexposed_cursor_input_columns, -}; -use crate::bindings::_helpers::planner_strategy::lineage::reject_capped_watermark_inputs; -use crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_configured_microbatch_limits; -use crate::bindings::_helpers::planner_strategy::model_entries::assemble_model_plan_entry; -use crate::bindings::_helpers::planner_strategy::strategy::{ - materialization_of, resolve_model_plan_action, +use crate::planner_entries::models::{ + ModelPlanContext, PlanEntriesRequest, PlanEntryBuildInputs, PlannerRelationsContext, }; -use crate::bindings::_helpers::planner_strategy::values::text; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::planner_phases::main::inputs::class_field::class_field; +use crate::planner_phases::main::rebuilds::check_full_rebuild_allowed::check_full_rebuild_allowed; +use crate::planner_phases::main::rebuilds::describe_full_rebuild_cause::describe_full_rebuild_cause; +use crate::planner_phases::main::rebuilds::resolve_model_full_refresh::resolve_model_full_refresh; +use crate::planner_phases::models::ModelEntryResults; +use crate::planner_resolution::main::cursor::config_str::config_str; +use crate::planner_resolution::main::cursor::cursor_safety_configs::cursor_safety_configs; +use crate::planner_resolution::main::cursor::empty_cursor_inputs::empty_cursor_inputs; +use crate::planner_resolution::main::cursor::microbatch_range::microbatch_range; +use crate::planner_resolution::main::cursor::model_cursor_overrides::model_cursor_overrides; +use crate::planner_resolution::main::cursor::plan_cursor_bounds::plan_cursor_bounds; +use crate::planner_resolution::main::cursor::with_cursor_safety_warnings::with_cursor_safety_warnings; +use crate::planner_resolution::main::references::model_hook_entries::model_hook_entries; +use crate::planner_resolution::main::references::resolve_model_sql::resolve_model_sql; +use crate::planner_resolution::models::CursorWindowInputs; +use crate::planner_resolution::models::ModelSqlRequest as SqlRequest; +use crate::planner_resolution::models::Resolution; +use crate::planner_strategy::main::cursor_inputs::build_cursor_input_relations::build_cursor_input_relations; +use crate::planner_strategy::main::cursor_inputs::reject_unexposed_cursor_input_columns::reject_unexposed_cursor_input_columns; +use crate::planner_strategy::main::cursor_inputs::runtime_cursor_producer_names::runtime_cursor_producer_names; +use crate::planner_strategy::main::lineage::reject_capped_watermark_inputs::reject_capped_watermark_inputs; +use crate::planner_strategy::main::microbatch_limits::apply_configured_microbatch_limits::apply_configured_microbatch_limits; +use crate::planner_strategy::main::model_entries::assemble_model_plan_entry::assemble_model_plan_entry; +use crate::planner_strategy::main::strategy::materialization_of::materialization_of; +use crate::planner_strategy::main::strategy::resolve_model_plan_action::resolve_model_plan_action; +use crate::planner_strategy::main::values::text::text; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; @@ -52,7 +51,7 @@ use sqlbuild_python_project::semantic_shapes::main::source_binding_diagnostics:: const MICROBATCH_COMPLEXITY_CODE: &str = "MICROBATCH_CONCURRENCY_COMPLEXITY"; /// The `ModelPlanContext` fields every model's planning reads, read once per plan. -struct ContextReads<'py> { +pub(crate) struct ContextReads<'py> { context: Bound<'py, PyAny>, lookups: CursorInputLookups<'py>, resolution: Resolution<'py>, @@ -60,7 +59,7 @@ struct ContextReads<'py> { } impl<'py> ContextReads<'py> { - fn of( + pub(crate) fn of( context: Bound<'py, PyAny>, adapter: Bound<'py, PyAny>, external_resolver: Bound<'py, PyAny>, @@ -86,19 +85,19 @@ impl<'py> ContextReads<'py> { } /// One model's planning request after change detection. -struct ModelPlanRequest<'a, 'py> { - model: Bound<'py, PyAny>, - snapshot: Bound<'py, PyAny>, - context: &'a ContextReads<'py>, - sql_analysis_enabled: bool, - full_refresh: bool, - overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), - change_result: Bound<'py, PyAny>, - protect_full_refresh: bool, +pub(crate) struct ModelPlanRequest<'a, 'py> { + pub(crate) model: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) context: &'a ContextReads<'py>, + pub(crate) sql_analysis_enabled: bool, + pub(crate) full_refresh: bool, + pub(crate) overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) change_result: Bound<'py, PyAny>, + pub(crate) protect_full_refresh: bool, } /// Build a model plan entry and its warnings from a resolved change result. -fn plan_model_from_change<'py>( +pub(crate) fn plan_model_from_change<'py>( request: &ModelPlanRequest<'_, 'py>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let model = &request.model; @@ -540,69 +539,6 @@ pub(crate) fn plan_entries<'py>( Ok(ModelEntryResults { entries, warnings }) } -/// Detect changes and build a plan entry and warnings for a single model. -#[pyfunction] -#[pyo3(signature = (**kwargs))] -pub(crate) fn plan_model<'py>( - py: Python<'py>, - kwargs: Option>, -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - let args = KeywordArguments::read( - py, - "plan_model", - kwargs, - ( - &[ - "model", - "snapshot", - "adapter", - "context", - "sql_analysis_enabled", - "query_change_tracking", - "full_refresh", - "cursor_overrides", - ], - &["external_sql_reference_resolver"], - ), - )?; - let model = args.value("model")?; - let snapshot = args.value("snapshot")?; - let change_result = detect_model_changes( - py, - Some(keywords( - py, - &[ - ("model", model.clone()), - ("snapshot", snapshot.clone()), - ("sql_analysis_enabled", args.value("sql_analysis_enabled")?), - ( - "query_change_tracking", - args.value("query_change_tracking")?, - ), - ("full_refresh", args.value("full_refresh")?), - ], - )?), - )?; - let overrides = args.value("cursor_overrides")?; - plan_model_from_change(&ModelPlanRequest { - model, - snapshot, - context: &ContextReads::of( - args.value("context")?, - args.value("adapter")?, - args.value("external_sql_reference_resolver")?, - )?, - sql_analysis_enabled: args.flag("sql_analysis_enabled")?, - full_refresh: args.flag("full_refresh")?, - overrides: ( - native_field(&overrides, "start_cursor_override")?, - native_field(&overrides, "end_cursor_override")?, - ), - change_result, - protect_full_refresh: true, - }) -} - /// Check if any scope dependency is in the selected keys. #[pyfunction] #[pyo3(signature = (*, scope_deps, selected_keys))] @@ -617,8 +553,3 @@ pub(crate) fn scope_overlaps( } Ok(false) } - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(plan_model, module)?)?; - module.add_function(wrap_pyfunction!(scope_overlaps, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs index efaf15b9fa..e6525f08cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs @@ -1,21 +1,19 @@ //! Relation locations and source metadata for plan entry construction. -use crate::bindings::_helpers::planner_entries::models::{ - PlannerRelationsContext, RelationsRequest, -}; -use crate::bindings::_helpers::planner_entries::values::planner_model; -use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; -use crate::bindings::_helpers::planner_resolution::locations::build_model_locations; -use crate::bindings::_helpers::planner_resolution::locations::build_seed_locations; -use crate::bindings::_helpers::planner_resolution::refs::apply_deferred_locations; -use crate::bindings::_helpers::planner_strategy::audit_entries::audit_is_selected; -use crate::bindings::_helpers::planner_warehouse::semantic_sources::semantic_source_columns; -use crate::bindings::_helpers::planner_warehouse::source_columns::gather_source_inspection; -use crate::bindings::_helpers::planner_warehouse::source_deferral::build_source_read_map; -use crate::bindings::_helpers::planner_warehouse::source_deferral::with_declared_source_reads; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::planner_entries::_helpers::entries::values::planner_model; +use crate::planner_entries::models::{PlannerRelationsContext, RelationsRequest}; +use crate::planner_resolution::main::references::apply_deferred_locations::apply_deferred_locations; +use crate::planner_resolution::main::references::build_function_locations::build_function_locations; +use crate::planner_resolution::main::references::build_model_locations::build_model_locations; +use crate::planner_resolution::main::references::build_seed_locations::build_seed_locations; +use crate::planner_strategy::main::audit_entries::audit_is_selected::audit_is_selected; +use crate::planner_warehouse::main::sources::build_source_read_map::build_source_read_map; +use crate::planner_warehouse::main::sources::gather_source_inspection::gather_source_inspection; +use crate::planner_warehouse::main::sources::semantic_source_columns::semantic_source_columns; +use crate::planner_warehouse::main::sources::with_declared_source_reads::with_declared_source_reads; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -219,7 +217,7 @@ pub(crate) fn planner_relations_context<'py>( /// Resolve relation locations and source metadata for plan entry construction. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn build_planner_relations_context<'py>( +pub(crate) fn build_planner_relations_context<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -250,7 +248,3 @@ fn build_planner_relations_context<'py>( relation_keys: args.value("relation_keys")?, }) } - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_planner_relations_context, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/values.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/values.rs index e8195c4f43..0e93c43efb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/values.rs @@ -11,11 +11,11 @@ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_key use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -pub(super) const MICROBATCH_CONCURRENCY_MESSAGE: &str = +pub(crate) const MICROBATCH_CONCURRENCY_MESSAGE: &str = "batch_concurrency > 1 requires concurrent microbatches; selected model(s): "; /// The native record `name(**pairs)`. -pub(super) fn native<'py>( +pub(crate) fn native<'py>( py: Python<'py>, name: RecordId, pairs: &[(&str, Bound<'py, PyAny>)], @@ -24,7 +24,7 @@ pub(super) fn native<'py>( } /// The planner record `name(**pairs)`. -pub(super) fn planner_model<'py>( +pub(crate) fn planner_model<'py>( py: Python<'py>, name: RecordId, pairs: &[(&str, Bound<'py, PyAny>)], @@ -33,7 +33,7 @@ pub(super) fn planner_model<'py>( } /// One member of a planner enumeration. -pub(super) fn planner_type<'py>( +pub(crate) fn planner_type<'py>( py: Python<'py>, class: RecordId, member: &str, @@ -42,22 +42,22 @@ pub(super) fn planner_type<'py>( } /// A Python string. -pub(super) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { +pub(crate) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { PyString::new(py, text).into_any() } /// A Python bool. -pub(super) fn boolean(py: Python<'_>, value: bool) -> PyResult> { +pub(crate) fn boolean(py: Python<'_>, value: bool) -> PyResult> { Ok(value.into_pyobject(py)?.to_owned().into_any()) } /// `None`. -pub(super) fn none(py: Python<'_>) -> Bound<'_, PyAny> { +pub(crate) fn none(py: Python<'_>) -> Bound<'_, PyAny> { py.None().into_bound(py) } /// `dataclasses.replace(object, **fields)`. -pub(super) fn replaced<'py>( +pub(crate) fn replaced<'py>( object: &Bound<'py, PyAny>, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { @@ -65,7 +65,7 @@ pub(super) fn replaced<'py>( } /// `class(message, **fields)` from `module`, as a raised error. -pub(super) fn error_of( +pub(crate) fn error_of( py: Python<'_>, class: ErrorId, message: &str, diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/mod.rs new file mode 100644 index 0000000000..62e4ffc8ef --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod entries; +pub(crate) mod plans; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs index d958342e66..37cfc3b0c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs @@ -1,22 +1,24 @@ //! Clone planning: destination and origin entries, and deferred-clone graph traversal. -use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; -use crate::bindings::_helpers::planner_entries::values::{ +use crate::planner_entries::_helpers::entries::values::{ native, planner_model, planner_type, string, }; -use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; -use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; -use crate::bindings::_helpers::planner_resolution::locations::build_model_locations; -use crate::bindings::_helpers::planner_resolution::locations::build_seed_locations; -use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest; -use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; -use crate::bindings::_helpers::planner_resolution::orchestration::{Resolution, resolve_model_sql}; -use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; -use crate::bindings::_helpers::planner_warehouse::source_columns::gather_source_columns; +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_entries::types::PhaseTimings; +use crate::planner_identity::main::build_compiled_function_fingerprint_sql::build_compiled_function_fingerprint_sql; +use crate::planner_resolution::main::references::build_function_locations::build_function_locations; +use crate::planner_resolution::main::references::build_model_locations::build_model_locations; +use crate::planner_resolution::main::references::build_seed_locations::build_seed_locations; +use crate::planner_resolution::main::references::resolve_function_sql::resolve_function_sql; +use crate::planner_resolution::main::references::resolve_model_sql::resolve_model_sql; +use crate::planner_resolution::models::ModelSqlRequest; +use crate::planner_resolution::models::Resolution; +use crate::planner_strategy::main::strategy::materialization_of::materialization_of; +use crate::planner_warehouse::main::sources::gather_source_columns::gather_source_columns; use pyo3::PyRef; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::graph::main::downstream_edges::downstream_edges; use sqlbuild_core::graph_walk::main::clone_ancestors::clone_ancestors; use sqlbuild_core::graph_walk::models::ClonePolicy; @@ -53,9 +55,6 @@ const SOURCE_INSPECTION_PHASE: &str = "source_inspection"; /// The clone phase that plans entries from the compiled projects. const PLANNING_PHASE: &str = "planning"; -/// Each clone phase name with its seconds, in the order the phases ran. -pub(crate) type PhaseTimings = Vec<(String, f64)>; - /// Whether a model is explicitly disabled with a boolean `enabled = false`. fn disabled_model(model: &Bound<'_, PyAny>) -> PyResult { Ok(matches!( @@ -755,8 +754,3 @@ pub(crate) fn resolve_skipped_view_chain<'py>( )?; CloneGraph::read(&args)?.walk(true) } - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_clone_boundary, module)?)?; - module.add_function(wrap_pyfunction!(resolve_skipped_view_chain, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs similarity index 74% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs index 8e402988e3..88473cfaf3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs @@ -1,25 +1,20 @@ //! Static plans for commands that do not build models, and direct seed identity reads. -use crate::bindings::_helpers::planner_entries::models::RelationsRequest; -use crate::bindings::_helpers::planner_entries::plan_output::{ - function_entry, selected_test_entries, test_selected, -}; -use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; -use crate::bindings::_helpers::planner_entries::values::{ +use crate::planner_entries::_helpers::entries::values::{ boolean, none, planner_model, planner_type, }; -use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; -use crate::bindings::_helpers::planner_identity::metadata::build_seed_identity; -use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; -use crate::bindings::_helpers::planner_strategy::audit_entries::build_selected_audit_entries; -use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; -use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::planner_entries::_helpers::plans::plan_output::{ + function_entry, selected_test_entries, test_selected, +}; +use crate::planner_identity::main::build_compiled_function_fingerprint_sql::build_compiled_function_fingerprint_sql; +use crate::planner_identity::main::build_seed_identity::build_seed_identity; +use crate::planner_strategy::main::model_entries::extract_seed_columns::extract_seed_columns; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{ PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PySet, PySetMethods, PyString, PyTuple, }; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_SEED; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -27,8 +22,6 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; @@ -40,12 +33,12 @@ use std::collections::{BTreeMap, BTreeSet}; /// Optional fields of a static command plan; absent ones take `PlanOutput` defaults. #[derive(Default)] -struct StaticPlanParts<'py> { - execution_order: Option>, - fields: Vec<(&'static str, Bound<'py, PyAny>)>, +pub(crate) struct StaticPlanParts<'py> { + pub(crate) execution_order: Option>, + pub(crate) fields: Vec<(&'static str, Bound<'py, PyAny>)>, } -fn base_plan_output<'py>( +pub(crate) fn base_plan_output<'py>( project: &Bound<'py, PyAny>, scope: &Bound<'py, PyAny>, relations: &Bound<'py, PyAny>, @@ -99,106 +92,6 @@ fn base_plan_output<'py>( planner_model(py, RecordId::PlanOutput, &fields) } -/// Resolve command selection through the canonical build selector phase. -#[pyfunction] -#[pyo3(signature = (*, project, selection, auto_load_sources = false))] -pub(crate) fn resolve_static_command_scope<'py>( - project: &Bound<'py, PyAny>, - selection: &Bound<'py, PyAny>, - auto_load_sources: bool, -) -> PyResult> { - let py = project.py(); - resolve_planner_scopes( - project, - selection, - &planner_model( - py, - RecordId::PlannerPolicies, - &[("auto_load_sources", boolean(py, auto_load_sources)?)], - )?, - )? - .getattr("selected_scope") -} - -/// Resolve canonical locations and source routing without warehouse inventory. -#[pyfunction] -#[pyo3(signature = (**kwargs))] -pub(crate) fn resolve_static_relation_context<'py>( - py: Python<'py>, - kwargs: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "resolve_static_relation_context", - kwargs, - ( - &["project", "adapter", "scope"], - &[ - "deferral", - "project_config", - "local_config", - "relation_keys", - ], - ), - )?; - planner_relations_context(RelationsRequest { - project: args.value("project")?, - adapter: args.value("adapter")?, - connection: none(py), - scope: args.value("scope")?, - deferral: args.value("deferral")?, - project_config: args.value("project_config")?, - local_config: args.value("local_config")?, - known_source_columns: PyDict::new(py).into_any(), - relation_keys: args.value("relation_keys")?, - }) -} - -/// Project selected audits from canonical static state. -#[pyfunction] -#[pyo3(signature = (*, project, adapter, scope, relations))] -pub(crate) fn build_audit_command_plan<'py>( - project: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, - scope: &Bound<'py, PyAny>, - relations: &Bound<'py, PyAny>, -) -> PyResult> { - let py = project.py(); - let materializations = PyDict::new(py); - for model in compiled_models(project)?.try_iter()? { - let model = model?; - materializations.set_item( - name_object(&model)?, - native_field(&materialization_of(&model)?, "value")?, - )?; - } - let audits = object( - py, - build_selected_audit_entries(Some(&keywords( - py, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("scope", scope.clone()), - ("relations", relations.clone()), - ("model_materializations", materializations.into_any()), - ], - )?))?, - )?; - base_plan_output( - project, - scope, - relations, - StaticPlanParts { - execution_order: None, - fields: vec![( - "audit_entries", - PyTuple::new(py, audits.try_iter()?.collect::>>()?)?.into_any(), - )], - }, - ) -} - /// The tests' direct function dependencies and every project function those call. fn called_function_keys<'py>( project: &Bound<'py, PyAny>, @@ -402,15 +295,6 @@ pub(crate) fn build_seed_command_plan<'py>( ) } -/// Project static relation data for Python checks. -pub(crate) fn build_relation_command_plan<'py>( - project: &Bound<'py, PyAny>, - scope: &Bound<'py, PyAny>, - relations: &Bound<'py, PyAny>, -) -> PyResult> { - base_plan_output(project, scope, relations, StaticPlanParts::default()) -} - /// A schema target's sort key: the database with `None` first as an empty name, then the schema. type Target = (Option, String); @@ -513,12 +397,3 @@ pub(crate) fn read_selected_seed_fingerprints<'py>( } Ok(fingerprints) } - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_static_command_scope, module)?)?; - module.add_function(wrap_pyfunction!(resolve_static_relation_context, module)?)?; - module.add_function(wrap_pyfunction!(build_audit_command_plan, module)?)?; - module.add_function(wrap_pyfunction!(build_test_command_plan, module)?)?; - module.add_function(wrap_pyfunction!(build_seed_command_plan, module)?)?; - module.add_function(wrap_pyfunction!(read_selected_seed_fingerprints, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs similarity index 61% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs index 0a182d7c85..4a069d1b96 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs @@ -1,14 +1,11 @@ //! Display-only plans for external plan surfaces when warehouse planning cannot resolve. -use crate::bindings::_helpers::planner_entries::values::{ +use crate::planner_entries::_helpers::entries::values::{ boolean, native, none, planner_model, planner_type, string, }; -use crate::bindings::_helpers::planner_phases::full_refresh::resolve_model_full_refresh; -use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::graph::main::downstream_edges::downstream_edges; +use crate::planner_phases::main::rebuilds::resolve_model_full_refresh::resolve_model_full_refresh; +use crate::planner_strategy::main::strategy::materialization_of::materialization_of; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; @@ -16,10 +13,6 @@ use sqlbuild_python_front::model_config::main::config::model_values::model_value use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; -use sqlbuild_python_project::graph::main::project::edge_dict::edge_dict; -use sqlbuild_python_project::graph::models::KeyCache; -use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; use sqlbuild_python_project::planner_phases::main::object::object; const VIEW: &str = "view"; @@ -51,7 +44,7 @@ fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult( +pub(crate) fn display_entry<'py>( model: &Bound<'py, PyAny>, full_refresh: bool, ) -> PyResult> { @@ -129,50 +122,3 @@ fn display_entry<'py>( ], ) } - -/// Build a display-only plan when warehouse planning cannot fully resolve. -#[pyfunction] -#[pyo3(signature = (*, project, selected_model_names, full_refresh))] -pub(crate) fn build_display_only_sqlbuild_plan<'py>( - project: &Bound<'py, PyAny>, - selected_model_names: &Bound<'py, PyAny>, - full_refresh: bool, -) -> PyResult> { - let py: Python<'py> = project.py(); - let selected = PyFrozenSet::new( - py, - selected_model_names - .try_iter()? - .collect::>>()?, - )?; - let mut entries: Vec> = Vec::new(); - for model in compiled_models(project)?.try_iter()? { - let model = model?; - if selected.contains(name_object(&model)?)? { - entries.push(display_entry(&model, full_refresh)?); - } - } - let keys: Vec> = entries - .iter() - .map(|entry| native_field(entry, "key")) - .collect::>()?; - let upstream = execution_deps(project)?; - let cache = KeyCache::new(py)?; - let downstream = edge_dict(&cache, &downstream_edges(&upstream))?.into_any(); - let upstream = edge_dict(&cache, &upstream)?.into_any(); - planner_model( - py, - RecordId::PlanOutput, - &[ - ("execution_order", PyTuple::new(py, &keys)?.into_any()), - ("model_entries", PyTuple::new(py, entries)?.into_any()), - ("selected_keys", PyFrozenSet::new(py, keys)?.into_any()), - ("upstream_deps", upstream), - ("downstream_deps", downstream), - ], - ) -} - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_display_only_sqlbuild_plan, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs similarity index 59% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs index 1599f248c9..1be05282ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs @@ -3,53 +3,44 @@ use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::planner_changes::actions::resolve_model_actions; -use crate::bindings::_helpers::planner_changes::detect::detect_changes; -use crate::bindings::_helpers::planner_entries::test_planning::{ - PlanningRequest, SqlTestPlanning, start_planning, -}; -use crate::bindings::_helpers::planner_entries::values::{native, replaced}; -use crate::bindings::_helpers::planner_phases::buildability::check_selected_scope_buildability; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::deferred_origins::defer_to_migration_origins; -use crate::bindings::_helpers::planner_phases::execution_changes::{ - build_planner_entry_results, reconcile_execution_changes, -}; -use crate::bindings::_helpers::planner_phases::identities::{ - build_planner_identity_context, detect_stale_warning_changes, -}; -use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; -use crate::bindings::_helpers::planner_phases::models::PlannerRuntime; -use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; -use crate::bindings::_helpers::planner_phases::models::{ +use crate::planner_changes::main::detect_changes::detect_changes; +use crate::planner_changes::main::resolve_model_actions::resolve_model_actions; +use crate::planner_entries::_helpers::entries::values::{native, replaced}; +use crate::planner_entries::_helpers::plans::test_planning::SqlTestPlanning; +use crate::planner_phases::main::inputs::class_field::class_field; +use crate::planner_phases::main::inputs::gather_planner_warehouse_state::gather_planner_warehouse_state; +use crate::planner_phases::main::inputs::warehouse_field::warehouse_field; +use crate::planner_phases::main::outputs::add_deferred_origin_notes::add_deferred_origin_notes; +use crate::planner_phases::main::outputs::add_plan_metadata::add_plan_metadata; +use crate::planner_phases::main::outputs::add_plan_warnings::add_plan_warnings; +use crate::planner_phases::main::outputs::add_storage_policies::add_storage_policies; +use crate::planner_phases::main::outputs::base_plan_output::base_plan_output; +use crate::planner_phases::main::outputs::build_planner_identity_context::build_planner_identity_context; +use crate::planner_phases::main::outputs::check_selected_scope_buildability::check_selected_scope_buildability; +use crate::planner_phases::main::outputs::detect_stale_warning_changes::detect_stale_warning_changes; +use crate::planner_phases::main::outputs::unselected_upstream_models::unselected_upstream_models; +use crate::planner_phases::main::rebuilds::build_planner_entry_results::build_planner_entry_results; +use crate::planner_phases::main::rebuilds::defer_to_migration_origins::defer_to_migration_origins; +use crate::planner_phases::main::rebuilds::reconcile_execution_changes::reconcile_execution_changes; +use crate::planner_phases::models::PlannerIdentityContext; +use crate::planner_phases::models::{ PlanOutputContext, PlanWarningContext, PlannedModelEntries, StoragePolicyContext, }; -use crate::bindings::_helpers::planner_phases::output_assembly::unselected_upstream_models; -use crate::bindings::_helpers::planner_phases::output_assembly::{ - add_deferred_origin_notes, add_plan_metadata, add_plan_warnings, add_storage_policies, - base_plan_output, -}; -use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; -use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; -use crate::bindings::_helpers::planner_phases::warehouse_state::gather_planner_warehouse_state; -use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; -use crate::bindings::_helpers::planner_warehouse::source_freshness::build_planner_source_freshness_result; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::planner_resolution::main::cursor::type_cursor_overrides::type_cursor_overrides; +use crate::planner_warehouse::main::sources::build_planner_source_freshness_result::build_planner_source_freshness_result; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::build_command::main::utc_now::utc_now; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_scope::main::resolve_planner_scopes::resolve_planner_scopes; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use std::time::Instant; -const INSPECTION_CATALOG: AdapterHost = AdapterHost::OpenInspectionCatalog; +pub(crate) const INSPECTION_CATALOG: AdapterHost = AdapterHost::OpenInspectionCatalog; /// Selected scopes, and the overrides with cursor overrides typed for the selected models. -fn scopes_and_cursor_overrides<'py>( +pub(crate) fn scopes_and_cursor_overrides<'py>( args: &KeywordArguments<'py>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let project = args.value("project")?; @@ -141,7 +132,7 @@ fn deferred_migration_origins<'py>( } /// The plan phases after warehouse inspection, run inside the inspection catalog. -fn planned_output<'py>( +pub(crate) fn planned_output<'py>( args: &KeywordArguments<'py>, runtime: &Bound<'py, PyAny>, (scopes, overrides): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -234,75 +225,3 @@ fn planned_output<'py>( } Ok(plan_output) } - -/// Build the execution plan for one selection against live warehouse state. -#[pyfunction] -#[pyo3(signature = (**kwargs))] -pub(crate) fn build_execution_plan<'py>( - py: Python<'py>, - kwargs: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "build_execution_plan", - kwargs, - ( - &[ - "project", - "adapter", - "connection", - "selection", - "overrides", - "deferral", - "policies", - ], - &["on_progress", "project_config", "local_config"], - ), - )?; - let project = args.value("project")?; - let adapter = args.value("adapter")?; - let connection = args.value("connection")?; - let runtime = Bound::new( - py, - PlannerRuntime { - project: project.clone().unbind(), - adapter: adapter.clone().unbind(), - connection: connection.clone().unbind(), - project_config: args.value("project_config")?.unbind(), - local_config: args.value("local_config")?.unbind(), - on_progress: args.value("on_progress")?.unbind(), - invocation_time: utc_now(py)?.unbind(), - }, - )? - .into_any(); - let (scopes, overrides) = scopes_and_cursor_overrides(&args)?; - let inspection = class_field(&scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?; - let test_planning = start_planning( - py, - PlanningRequest { - project: project.clone().unbind(), - adapter: adapter.clone().unbind(), - selected_keys: native_field(&inspection, "selected_keys")?.unbind(), - sql_test_selection: native_field(&inspection, "sql_test_selection")?.unbind(), - }, - native_field(&args.value("policies")?, "plan_sql_tests")?.is_truthy()?, - )?; - let planned = (|| { - let catalog = call_with( - &host_entry(py, INSPECTION_CATALOG)?, - &[("adapter", adapter), ("connection", connection)], - )?; - within(&catalog, |_| { - planned_output(&args, &runtime, (&scopes, &overrides), test_planning.get()) - }) - })(); - let joined = test_planning.get().result(py); - let planned = planned?; - joined?; - Ok(planned) -} - -/// Add the execution plan entry point. -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_execution_plan, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/mod.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/mod.rs new file mode 100644 index 0000000000..a7993b715c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod clone_plans; +pub(crate) mod command_plans; +pub(crate) mod display_plan; +pub(crate) mod execution_plan; +pub(crate) mod plan_output; +pub(crate) mod test_planning; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs index ee41bcdd48..327003a020 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs @@ -1,16 +1,16 @@ //! `PlanOutput` assembly from planned model entries, seeds, sources, functions, audits and tests. -use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps; -use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; -use crate::bindings::_helpers::planner_entries::values::{ +use crate::planner_entries::_helpers::entries::model_entries::scope_overlaps; +use crate::planner_entries::_helpers::entries::values::{ error_of, native, none, planner_model, planner_type, string, }; -use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; -use crate::bindings::_helpers::planner_strategy::audit_entries::build_selected_audit_entries; -use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::planner_entries::models::PlanOutputInputs; +use crate::planner_resolution::main::references::resolve_function_sql::resolve_function_sql; +use crate::planner_strategy::main::audit_entries::build_selected_audit_entries::build_selected_audit_entries; +use crate::planner_strategy::main::model_entries::extract_seed_columns::extract_seed_columns; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::record_class::record_class; @@ -36,7 +36,6 @@ use sqlbuild_python_project::resource_views::main::function_reads::source_file_p use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; use sqlbuild_python_project::sql_test_glue::main::assemble_sql_test_plans::assemble_sql_test_plans; -use sqlbuild_python_project::sql_test_glue::main::sql_test_model_chain_names_by_key_function::sql_test_model_chain_names_by_key_function; use sqlbuild_python_project::sql_test_glue::models::SqlTestAssembly; const LOADER_CONSUMER: &str = "loader"; @@ -347,7 +346,7 @@ fn require_intermediate_loader_targets<'py>( } /// Build one canonical function entry from resolved relation inputs. -pub(super) fn function_entry<'py>( +pub(crate) fn function_entry<'py>( function: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, relations: &Bound<'py, PyAny>, @@ -463,7 +462,7 @@ fn function_entries<'py>( } /// Whether a test is named by the selection or covers a selected resource, and not excluded. -pub(super) fn test_selected( +pub(crate) fn test_selected( test: &Bound<'_, PyAny>, selected_keys: &Bound<'_, PyAny>, selection: &Bound<'_, PyAny>, @@ -482,13 +481,13 @@ pub(super) fn test_selected( } /// Planned SQL test entries and the warnings planning them produced. -pub(super) struct SelectedTests<'py> { +pub(crate) struct SelectedTests<'py> { pub(super) entries: Vec>, pub(super) warnings: Vec>, } /// Selected SQL test entries and their warnings, failing on invalid fixtures. -pub(super) fn selected_test_entries<'py>( +pub(crate) fn selected_test_entries<'py>( (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), selected_keys: &Bound<'py, PyAny>, selection: &Bound<'py, PyAny>, @@ -772,7 +771,7 @@ pub(crate) fn plan_selected_sql_tests<'py>( /// Selected SQL test entries and warnings, optionally narrowed to one parameterized case. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn build_selected_test_entries<'py>( +pub(crate) fn build_selected_test_entries<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult<(Bound<'py, PyList>, Bound<'py, PyList>)> { @@ -796,8 +795,3 @@ fn build_selected_test_entries<'py>( PyList::new(py, selected.warnings)?, )) } - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(sql_test_model_chain_names_by_key_function(module)?)?; - module.add_function(wrap_pyfunction!(build_selected_test_entries, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs rename to crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs index 083a50daba..1fbe363f6f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs @@ -6,16 +6,13 @@ use std::sync::{Mutex, PoisonError}; use std::time::{Duration, Instant}; use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::PyDict; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::{pyclass, pyfunction, pymethods}; -use crate::bindings::_helpers::planner_entries::plan_output::plan_selected_sql_tests; +use crate::planner_entries::_helpers::plans::plan_output::plan_selected_sql_tests; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_threads::main::signal_interpreter_exit::signal_interpreter_exit; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -215,7 +212,7 @@ pub(crate) fn start_planning( /// Join SQL test planning threads at exit: finalization ending one inside native planning aborts. #[pyfunction] -fn join_sql_test_planning_threads(py: Python<'_>) -> PyResult<()> { +pub(crate) fn join_sql_test_planning_threads(py: Python<'_>) -> PyResult<()> { signal_interpreter_exit(); let threads: &PlanningThreads = planning_threads(py)?.get(); py.detach(|| threads.wait(PLANNING_EXIT_JOIN)); @@ -225,7 +222,7 @@ fn join_sql_test_planning_threads(py: Python<'_>) -> PyResult<()> { /// Start SQL test planning on a native thread, or resolve to no tests when disabled. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn start_sql_test_planning<'py>( +pub(crate) fn start_sql_test_planning<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -255,13 +252,3 @@ fn start_sql_test_planning<'py>( args.flag("enabled")?, ) } - -/// Add the planning handle and entry points, and join planning threads at interpreter exit. -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(start_sql_test_planning, module)?)?; - let hook = wrap_pyfunction!(join_sql_test_planning_threads, module)?; - module.add_function(hook.clone())?; - imported(module.py(), "atexit", "register")?.call1((hook,))?; - Ok(()) -} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/constants.rs b/crates/sqlbuild-python-planner/src/planner_entries/constants.rs new file mode 100644 index 0000000000..c55b87786f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/constants.rs @@ -0,0 +1,26 @@ +//! Public constants of planner entries. + +use crate::planner_entries::_helpers::entries::getters::read_model_plan_context; +use crate::planner_entries::_helpers::entries::getters::read_plan_entry_build_inputs; +use crate::planner_entries::_helpers::entries::getters::read_planner_relations_context; +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_entries::models::PlanEntryBuildInputs; +use crate::planner_entries::models::PlannerRelationsContext; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_model_plan_context, + }, + GetterEntry { + class: class_of::, + read: read_plan_entry_build_inputs, + }, + GetterEntry { + class: class_of::, + read: read_planner_relations_context, + }, +]; diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/default_microbatch_limit_action.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/default_microbatch_limit_action.rs new file mode 100644 index 0000000000..931c88d0f6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/default_microbatch_limit_action.rs @@ -0,0 +1,10 @@ +//! The microbatch limit action used when a project configures none. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::types::RecordId; + +/// The microbatch limit action used when a project configures none. +pub fn default_microbatch_limit_action(py: Python<'_>) -> PyResult> { + enum_member(py, RecordId::MicrobatchLimitAction, "ERROR") +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/mod.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/mod.rs new file mode 100644 index 0000000000..86fd6222a7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/mod.rs @@ -0,0 +1,6 @@ +pub mod default_microbatch_limit_action; +pub mod plan_entries; +pub mod plan_model; +pub mod planner_relations_context; +pub mod register_model_entries; +pub mod register_relations; diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_entries.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_entries.rs new file mode 100644 index 0000000000..04fa332dc4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_entries.rs @@ -0,0 +1,12 @@ +//! Build model plan entries from snapshot and per-model resolved actions. + +use crate::planner_entries::_helpers::entries::model_entries; +use crate::planner_entries::models::PlanEntriesRequest; +use crate::planner_phases::models::ModelEntryResults; +use pyo3::prelude::PyResult; + +/// Build model plan entries from snapshot and per-model resolved actions. +#[inline] +pub fn plan_entries<'py>(request: PlanEntriesRequest<'py>) -> PyResult> { + model_entries::plan_entries(request) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_model.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_model.rs new file mode 100644 index 0000000000..de7eebbf54 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/plan_model.rs @@ -0,0 +1,75 @@ +//! Detect changes and build a plan entry and warnings for a single model. + +use crate::planner_changes::main::detect_model_changes::detect_model_changes; +use crate::planner_entries::_helpers::entries::model_entries::ContextReads; +use crate::planner_entries::_helpers::entries::model_entries::ModelPlanRequest; +use crate::planner_entries::_helpers::entries::model_entries::plan_model_from_change; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Detect changes and build a plan entry and warnings for a single model. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub fn plan_model<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let args = KeywordArguments::read( + py, + "plan_model", + kwargs, + ( + &[ + "model", + "snapshot", + "adapter", + "context", + "sql_analysis_enabled", + "query_change_tracking", + "full_refresh", + "cursor_overrides", + ], + &["external_sql_reference_resolver"], + ), + )?; + let model = args.value("model")?; + let snapshot = args.value("snapshot")?; + let change_result = detect_model_changes( + py, + Some(keywords( + py, + &[ + ("model", model.clone()), + ("snapshot", snapshot.clone()), + ("sql_analysis_enabled", args.value("sql_analysis_enabled")?), + ( + "query_change_tracking", + args.value("query_change_tracking")?, + ), + ("full_refresh", args.value("full_refresh")?), + ], + )?), + )?; + let overrides = args.value("cursor_overrides")?; + plan_model_from_change(&ModelPlanRequest { + model, + snapshot, + context: &ContextReads::of( + args.value("context")?, + args.value("adapter")?, + args.value("external_sql_reference_resolver")?, + )?, + sql_analysis_enabled: args.flag("sql_analysis_enabled")?, + full_refresh: args.flag("full_refresh")?, + overrides: ( + native_field(&overrides, "start_cursor_override")?, + native_field(&overrides, "end_cursor_override")?, + ), + change_result, + protect_full_refresh: true, + }) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/planner_relations_context.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/planner_relations_context.rs new file mode 100644 index 0000000000..cb9f74ca22 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/planner_relations_context.rs @@ -0,0 +1,13 @@ +//! Resolve relation locations and source metadata for plan entry construction. + +use crate::planner_entries::_helpers::entries::relations; +use crate::planner_entries::models::RelationsRequest; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Resolve relation locations and source metadata for plan entry construction. +#[inline] +pub fn planner_relations_context<'py>( + request: RelationsRequest<'py>, +) -> PyResult> { + relations::planner_relations_context(request) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_model_entries.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_model_entries.rs new file mode 100644 index 0000000000..7a6254ff28 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_model_entries.rs @@ -0,0 +1,12 @@ +//! Register the model entries functions and classes. + +use crate::planner_entries::_helpers::entries::model_entries::scope_overlaps; +use crate::planner_entries::main::entries::plan_model::plan_model; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the model entries functions and classes. +pub fn register_model_entries(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(plan_model, module)?)?; + module.add_function(wrap_pyfunction!(scope_overlaps, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_relations.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_relations.rs new file mode 100644 index 0000000000..741b300888 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/entries/register_relations.rs @@ -0,0 +1,10 @@ +//! Register the relations functions and classes. + +use crate::planner_entries::_helpers::entries::relations::build_planner_relations_context; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the relations functions and classes. +pub fn register_relations(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_planner_relations_context, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/mod.rs new file mode 100644 index 0000000000..81d411af3f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/mod.rs @@ -0,0 +1,2 @@ +pub mod entries; +pub mod plans; diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs new file mode 100644 index 0000000000..9ae83b6857 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs @@ -0,0 +1,59 @@ +//! Project selected audits from canonical static state. + +use crate::planner_entries::_helpers::plans::command_plans::StaticPlanParts; +use crate::planner_entries::_helpers::plans::command_plans::base_plan_output; +use crate::planner_strategy::main::audit_entries::build_selected_audit_entries::build_selected_audit_entries; +use crate::planner_strategy::main::strategy::materialization_of::materialization_of; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::planner_phases::main::object::object; + +/// Project selected audits from canonical static state. +#[pyfunction] +#[pyo3(signature = (*, project, adapter, scope, relations))] +pub fn build_audit_command_plan<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let materializations = PyDict::new(py); + for model in compiled_models(project)?.try_iter()? { + let model = model?; + materializations.set_item( + name_object(&model)?, + native_field(&materialization_of(&model)?, "value")?, + )?; + } + let audits = object( + py, + build_selected_audit_entries(Some(&keywords( + py, + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("scope", scope.clone()), + ("relations", relations.clone()), + ("model_materializations", materializations.into_any()), + ], + )?))?, + )?; + base_plan_output( + project, + scope, + relations, + StaticPlanParts { + execution_order: None, + fields: vec![( + "audit_entries", + PyTuple::new(py, audits.try_iter()?.collect::>>()?)?.into_any(), + )], + }, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_display_only_sqlbuild_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_display_only_sqlbuild_plan.rs new file mode 100644 index 0000000000..b030728547 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_display_only_sqlbuild_plan.rs @@ -0,0 +1,58 @@ +//! Build a display-only plan when warehouse planning cannot fully resolve. + +use crate::planner_entries::_helpers::entries::values::planner_model; +use crate::planner_entries::_helpers::plans::display_plan::display_entry; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyFrozenSet, PyTuple}; +use sqlbuild_analysis::graph::main::downstream_edges::downstream_edges; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::graph::main::project::edge_dict::edge_dict; +use sqlbuild_python_project::graph::models::KeyCache; +use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; + +/// Build a display-only plan when warehouse planning cannot fully resolve. +#[pyfunction] +#[pyo3(signature = (*, project, selected_model_names, full_refresh))] +pub fn build_display_only_sqlbuild_plan<'py>( + project: &Bound<'py, PyAny>, + selected_model_names: &Bound<'py, PyAny>, + full_refresh: bool, +) -> PyResult> { + let py: Python<'py> = project.py(); + let selected = PyFrozenSet::new( + py, + selected_model_names + .try_iter()? + .collect::>>()?, + )?; + let mut entries: Vec> = Vec::new(); + for model in compiled_models(project)?.try_iter()? { + let model = model?; + if selected.contains(name_object(&model)?)? { + entries.push(display_entry(&model, full_refresh)?); + } + } + let keys: Vec> = entries + .iter() + .map(|entry| native_field(entry, "key")) + .collect::>()?; + let upstream = execution_deps(project)?; + let cache = KeyCache::new(py)?; + let downstream = edge_dict(&cache, &downstream_edges(&upstream))?.into_any(); + let upstream = edge_dict(&cache, &upstream)?.into_any(); + planner_model( + py, + RecordId::PlanOutput, + &[ + ("execution_order", PyTuple::new(py, &keys)?.into_any()), + ("model_entries", PyTuple::new(py, entries)?.into_any()), + ("selected_keys", PyFrozenSet::new(py, keys)?.into_any()), + ("upstream_deps", upstream), + ("downstream_deps", downstream), + ], + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs new file mode 100644 index 0000000000..a854ea1cda --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs @@ -0,0 +1,85 @@ +//! Build the execution plan for one selection against live warehouse state. + +use crate::planner_entries::_helpers::plans::execution_plan::INSPECTION_CATALOG; +use crate::planner_entries::_helpers::plans::execution_plan::planned_output; +use crate::planner_entries::_helpers::plans::execution_plan::scopes_and_cursor_overrides; +use crate::planner_entries::_helpers::plans::test_planning::{PlanningRequest, start_planning}; +use crate::planner_phases::main::inputs::class_field::class_field; +use crate::planner_phases::models::PlannerRuntime; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_command::main::utc_now::utc_now; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; + +/// Build the execution plan for one selection against live warehouse state. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub fn build_execution_plan<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_execution_plan", + kwargs, + ( + &[ + "project", + "adapter", + "connection", + "selection", + "overrides", + "deferral", + "policies", + ], + &["on_progress", "project_config", "local_config"], + ), + )?; + let project = args.value("project")?; + let adapter = args.value("adapter")?; + let connection = args.value("connection")?; + let runtime = Bound::new( + py, + PlannerRuntime { + project: project.clone().unbind(), + adapter: adapter.clone().unbind(), + connection: connection.clone().unbind(), + project_config: args.value("project_config")?.unbind(), + local_config: args.value("local_config")?.unbind(), + on_progress: args.value("on_progress")?.unbind(), + invocation_time: utc_now(py)?.unbind(), + }, + )? + .into_any(); + let (scopes, overrides) = scopes_and_cursor_overrides(&args)?; + let inspection = class_field(&scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?; + let test_planning = start_planning( + py, + PlanningRequest { + project: project.clone().unbind(), + adapter: adapter.clone().unbind(), + selected_keys: native_field(&inspection, "selected_keys")?.unbind(), + sql_test_selection: native_field(&inspection, "sql_test_selection")?.unbind(), + }, + native_field(&args.value("policies")?, "plan_sql_tests")?.is_truthy()?, + )?; + let planned = (|| { + let catalog = call_with( + &host_entry(py, INSPECTION_CATALOG)?, + &[("adapter", adapter), ("connection", connection)], + )?; + within(&catalog, |_| { + planned_output(&args, &runtime, (&scopes, &overrides), test_planning.get()) + }) + })(); + let joined = test_planning.get().result(py); + let planned = planned?; + joined?; + Ok(planned) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_relation_command_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_relation_command_plan.rs new file mode 100644 index 0000000000..fb0a192aed --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_relation_command_plan.rs @@ -0,0 +1,14 @@ +//! Project static relation data for Python checks. + +use crate::planner_entries::_helpers::plans::command_plans::StaticPlanParts; +use crate::planner_entries::_helpers::plans::command_plans::base_plan_output; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Project static relation data for Python checks. +pub fn build_relation_command_plan<'py>( + project: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, +) -> PyResult> { + base_plan_output(project, scope, relations, StaticPlanParts::default()) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_seed_command_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_seed_command_plan.rs new file mode 100644 index 0000000000..5996a0b5bf --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_seed_command_plan.rs @@ -0,0 +1,15 @@ +//! Project selected direct seed work and canonical identities. + +use crate::planner_entries::_helpers::plans::command_plans; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Project selected direct seed work and canonical identities. +#[inline] +pub fn build_seed_command_plan<'py>( + project: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, + fingerprints: Option>, +) -> PyResult> { + command_plans::build_seed_command_plan(project, scope, relations, fingerprints) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_test_command_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_test_command_plan.rs new file mode 100644 index 0000000000..b5a78d1b67 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_test_command_plan.rs @@ -0,0 +1,14 @@ +//! Project selected SQL tests from canonical static state. + +use crate::planner_entries::_helpers::plans::command_plans; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyDict; + +/// Project selected SQL tests from canonical static state. +#[inline] +pub fn build_test_command_plan<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + command_plans::build_test_command_plan(py, kwargs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/mod.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/mod.rs new file mode 100644 index 0000000000..cef1b11857 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/mod.rs @@ -0,0 +1,17 @@ +pub mod build_audit_command_plan; +pub mod build_display_only_sqlbuild_plan; +pub mod build_execution_plan; +pub mod build_relation_command_plan; +pub mod build_seed_command_plan; +pub mod build_test_command_plan; +pub mod plan_output_fields; +pub mod read_selected_seed_fingerprints; +pub mod register_clone_plans; +pub mod register_command_plans; +pub mod register_display_plan; +pub mod register_execution_plan; +pub mod register_plan_output; +pub mod register_test_planning; +pub mod resolve_static_command_scope; +pub mod resolve_static_relation_context; +pub mod timed_clone_planning; diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/plan_output_fields.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/plan_output_fields.rs new file mode 100644 index 0000000000..d6add1d6d6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/plan_output_fields.rs @@ -0,0 +1,13 @@ +//! The plan output's fields from planned entries and resolved relation inputs. + +use crate::planner_entries::_helpers::plans::plan_output; +use crate::planner_entries::models::PlanOutputInputs; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// The plan output's fields from planned entries and resolved relation inputs. +#[inline] +pub fn plan_output_fields<'py>( + inputs: PlanOutputInputs<'py>, +) -> PyResult)>> { + plan_output::plan_output_fields(inputs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/read_selected_seed_fingerprints.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/read_selected_seed_fingerprints.rs new file mode 100644 index 0000000000..2a833b1c09 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/read_selected_seed_fingerprints.rs @@ -0,0 +1,15 @@ +//! Read only fingerprint state required by selected direct seeds. + +use crate::planner_entries::_helpers::plans::command_plans; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyDict; + +/// Read only fingerprint state required by selected direct seeds. +#[inline] +pub fn read_selected_seed_fingerprints<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + seeds: &Bound<'py, PyAny>, +) -> PyResult> { + command_plans::read_selected_seed_fingerprints(adapter, connection, seeds) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_clone_plans.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_clone_plans.rs new file mode 100644 index 0000000000..3f827aacd0 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_clone_plans.rs @@ -0,0 +1,12 @@ +//! Register the clone plans functions and classes. + +use crate::planner_entries::_helpers::plans::clone_plans::resolve_clone_boundary; +use crate::planner_entries::_helpers::plans::clone_plans::resolve_skipped_view_chain; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the clone plans functions and classes. +pub fn register_clone_plans(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_clone_boundary, module)?)?; + module.add_function(wrap_pyfunction!(resolve_skipped_view_chain, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_command_plans.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_command_plans.rs new file mode 100644 index 0000000000..3d5bd5f991 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_command_plans.rs @@ -0,0 +1,20 @@ +//! Register the command plans functions and classes. + +use crate::planner_entries::_helpers::plans::command_plans::build_seed_command_plan; +use crate::planner_entries::_helpers::plans::command_plans::build_test_command_plan; +use crate::planner_entries::_helpers::plans::command_plans::read_selected_seed_fingerprints; +use crate::planner_entries::main::plans::build_audit_command_plan::build_audit_command_plan; +use crate::planner_entries::main::plans::resolve_static_command_scope::resolve_static_command_scope; +use crate::planner_entries::main::plans::resolve_static_relation_context::resolve_static_relation_context; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the command plans functions and classes. +pub fn register_command_plans(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_static_command_scope, module)?)?; + module.add_function(wrap_pyfunction!(resolve_static_relation_context, module)?)?; + module.add_function(wrap_pyfunction!(build_audit_command_plan, module)?)?; + module.add_function(wrap_pyfunction!(build_test_command_plan, module)?)?; + module.add_function(wrap_pyfunction!(build_seed_command_plan, module)?)?; + module.add_function(wrap_pyfunction!(read_selected_seed_fingerprints, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_display_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_display_plan.rs new file mode 100644 index 0000000000..a248a2ee36 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_display_plan.rs @@ -0,0 +1,10 @@ +//! Register the display plan functions and classes. + +use crate::planner_entries::main::plans::build_display_only_sqlbuild_plan::build_display_only_sqlbuild_plan; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the display plan functions and classes. +pub fn register_display_plan(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_display_only_sqlbuild_plan, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_execution_plan.rs new file mode 100644 index 0000000000..1572e85575 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_execution_plan.rs @@ -0,0 +1,10 @@ +//! Add the execution plan entry point. + +use crate::planner_entries::main::plans::build_execution_plan::build_execution_plan; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Add the execution plan entry point. +pub fn register_execution_plan(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_execution_plan, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_plan_output.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_plan_output.rs new file mode 100644 index 0000000000..087d96c6f9 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_plan_output.rs @@ -0,0 +1,12 @@ +//! Register the plan output functions and classes. + +use crate::planner_entries::_helpers::plans::plan_output::build_selected_test_entries; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; +use sqlbuild_python_project::sql_test_glue::main::sql_test_model_chain_names_by_key_function::sql_test_model_chain_names_by_key_function; + +/// Register the plan output functions and classes. +pub fn register_plan_output(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(sql_test_model_chain_names_by_key_function(module)?)?; + module.add_function(wrap_pyfunction!(build_selected_test_entries, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_test_planning.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_test_planning.rs new file mode 100644 index 0000000000..be6dc52d85 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_test_planning.rs @@ -0,0 +1,18 @@ +//! Add the planning handle and entry points, and join planning threads at interpreter exit. + +use crate::planner_entries::_helpers::plans::test_planning::SqlTestPlanning; +use crate::planner_entries::_helpers::plans::test_planning::join_sql_test_planning_threads; +use crate::planner_entries::_helpers::plans::test_planning::start_sql_test_planning; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; +use sqlbuild_python_base::boundary::main::imported::imported; + +/// Add the planning handle and entry points, and join planning threads at interpreter exit. +pub fn register_test_planning(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(start_sql_test_planning, module)?)?; + let hook = wrap_pyfunction!(join_sql_test_planning_threads, module)?; + module.add_function(hook.clone())?; + imported(module.py(), "atexit", "register")?.call1((hook,))?; + Ok(()) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_command_scope.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_command_scope.rs new file mode 100644 index 0000000000..e9d0b5592e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_command_scope.rs @@ -0,0 +1,28 @@ +//! Resolve command selection through the canonical build selector phase. + +use crate::planner_entries::_helpers::entries::values::{boolean, planner_model}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_scope::main::resolve_planner_scopes::resolve_planner_scopes; + +/// Resolve command selection through the canonical build selector phase. +#[pyfunction] +#[pyo3(signature = (*, project, selection, auto_load_sources = false))] +pub fn resolve_static_command_scope<'py>( + project: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, + auto_load_sources: bool, +) -> PyResult> { + let py = project.py(); + resolve_planner_scopes( + project, + selection, + &planner_model( + py, + RecordId::PlannerPolicies, + &[("auto_load_sources", boolean(py, auto_load_sources)?)], + )?, + )? + .getattr("selected_scope") +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs new file mode 100644 index 0000000000..63cc2accfc --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs @@ -0,0 +1,43 @@ +//! Resolve canonical locations and source routing without warehouse inventory. + +use crate::planner_entries::_helpers::entries::relations::planner_relations_context; +use crate::planner_entries::_helpers::entries::values::none; +use crate::planner_entries::models::RelationsRequest; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// Resolve canonical locations and source routing without warehouse inventory. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub fn resolve_static_relation_context<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "resolve_static_relation_context", + kwargs, + ( + &["project", "adapter", "scope"], + &[ + "deferral", + "project_config", + "local_config", + "relation_keys", + ], + ), + )?; + planner_relations_context(RelationsRequest { + project: args.value("project")?, + adapter: args.value("adapter")?, + connection: none(py), + scope: args.value("scope")?, + deferral: args.value("deferral")?, + project_config: args.value("project_config")?, + local_config: args.value("local_config")?, + known_source_columns: PyDict::new(py).into_any(), + relation_keys: args.value("relation_keys")?, + }) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/timed_clone_planning.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/timed_clone_planning.rs new file mode 100644 index 0000000000..d4f4bc552b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/timed_clone_planning.rs @@ -0,0 +1,15 @@ +//! Plan a clone, with the seconds spent inspecting sources and planning. + +use crate::planner_entries::_helpers::plans::clone_plans; +use crate::planner_entries::types::PhaseTimings; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; + +/// Plan a clone, with the seconds spent inspecting sources and planning. +#[inline] +pub fn timed_clone_planning<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult<(Bound<'py, PyTuple>, PhaseTimings)> { + clone_plans::timed_clone_planning(py, kwargs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/mod.rs b/crates/sqlbuild-python-planner/src/planner_entries/mod.rs new file mode 100644 index 0000000000..37203f0548 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/mod.rs @@ -0,0 +1,7 @@ +//! Plan entry and plan output assembly over native planner resolution. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-planner/src/planner_entries/models.rs b/crates/sqlbuild-python-planner/src/planner_entries/models.rs new file mode 100644 index 0000000000..91867bcf9b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/models.rs @@ -0,0 +1,99 @@ +//! Public records of planner entries. + +use pyo3::prelude::{Bound, Py, PyAny}; +use pyo3::pyclass; +use sqlbuild_python_project::native_host::models::ModelLookup; + +/// What resolving relation locations and source metadata for one scope reads. +#[derive(Debug)] +pub struct RelationsRequest<'py> { + pub project: Bound<'py, PyAny>, + pub adapter: Bound<'py, PyAny>, + pub connection: Bound<'py, PyAny>, + pub scope: Bound<'py, PyAny>, + pub deferral: Bound<'py, PyAny>, + pub project_config: Bound<'py, PyAny>, + pub local_config: Bound<'py, PyAny>, + pub known_source_columns: Bound<'py, PyAny>, + pub relation_keys: Bound<'py, PyAny>, +} + +/// What building model plan entries for one execution scope reads. +#[derive(Debug)] +pub struct PlanEntriesRequest<'py> { + pub project: Bound<'py, PyAny>, + pub adapter: Bound<'py, PyAny>, + pub scope: Bound<'py, PyAny>, + pub snapshot: Bound<'py, PyAny>, + pub relations: Bound<'py, PyAny>, + pub resolved_models: Bound<'py, PyAny>, + pub cursor_overrides: Bound<'py, PyAny>, + pub full_refresh: bool, + pub build_inputs: Bound<'py, PyAny>, + /// The effective worker limit execution will use. + pub concurrency: i64, +} + +/// What assembling the plan output's fields reads. +#[derive(Debug)] +pub struct PlanOutputInputs<'py> { + pub project: Bound<'py, PyAny>, + pub adapter: Bound<'py, PyAny>, + pub scope: Bound<'py, PyAny>, + pub snapshot: Bound<'py, PyAny>, + pub relations: Bound<'py, PyAny>, + pub changes: Bound<'py, PyAny>, + pub model_entries: Vec>, + pub model_warnings: Vec>, + pub reload_sources: Bound<'py, PyAny>, + pub extras: Bound<'py, PyAny>, +} + +/// Resolved relation and source inputs for plan entry construction. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct PlannerRelationsContext { + pub model_locations: Py, + pub seed_locations: Py, + pub function_locations: Py, + pub source_map: Py, + pub source_read_map: Py, + pub python_source_read_map: Py, + pub source_warehouse_columns: Py, + pub star_exclude_keyword: Py, + pub listed_source_names: Py, +} + +/// Relation lookups and source columns for building one model plan entry. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct ModelPlanContext { + pub model_locations: Py, + pub models_by_name: ModelLookup, + pub seed_locations: Py, + pub function_locations: Py, + pub source_map: Py, + pub source_warehouse_columns: Py, + pub star_exclude_keyword: Py, + pub functions_by_name: Py, + pub runtime_cursor_producer_names: Py, + pub future_cursor_config: Py, + pub start_cursor_config: Py, + pub invocation_time: Py, +} + +/// Blocked models and cursor overrides for plan entry building. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct PlanEntryBuildInputs { + pub source_freshness_blocked_model_names: Py, + pub external_blocked_model_names: Py, + pub start_cursor_override: Py, + pub end_cursor_override: Py, + pub future_cursor_config: Py, + pub start_cursor_config: Py, + pub invocation_time: Py, + pub max_microbatches: Py, + pub max_microbatches_is_override: bool, + pub microbatch_limit_action: Py, +} diff --git a/crates/sqlbuild-python-planner/src/planner_entries/types.rs b/crates/sqlbuild-python-planner/src/planner_entries/types.rs new file mode 100644 index 0000000000..c437576b63 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_entries/types.rs @@ -0,0 +1,4 @@ +//! Public traits and type aliases of planner entries. + +/// Each clone phase name with its seconds, in the order the phases ran. +pub type PhaseTimings = Vec<(String, f64)>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs similarity index 86% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs rename to crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs index 13e30cf38e..68240b5a93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs @@ -1,10 +1,12 @@ //! Direct model version identities and honest write hashes over the planner scope. +use crate::planner_identity::main::hook_version_hashes::hook_version_hashes; +use crate::planner_identity::main::model_version_identity_metadata_json::model_version_identity_metadata_json; use std::collections::{HashMap, HashSet}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_identity::constants::{ MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE, TABLE_FN_RESOURCE, UDF_RESOURCE, @@ -13,13 +15,9 @@ use sqlbuild_analysis::planner_identity::main::expected_graph_identity_hashes::e use sqlbuild_analysis::planner_identity::main::graph_write_identity_hashes::graph_write_identity_hashes; use sqlbuild_analysis::planner_identity::models::{GraphKey, IdentityGraph, IdentityNode}; use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; -use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::planner_identity::metadata::{ - function_local_hashes, model_version_identity_metadata_json, seed_identity, -}; +use crate::planner_identity::_helpers::metadata::{function_local_hashes, seed_identity}; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; @@ -51,21 +49,6 @@ fn string_pairs(mapping: &Bound<'_, PyAny>) -> PyResult> { .collect() } -/// `{name: identity.version_hash}` of the discovered Python hooks. -pub(crate) fn hook_version_hashes<'py>( - hook_functions: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = hook_functions.py(); - let hashes: Bound<'py, PyDict> = PyDict::new(py); - let identities: Bound<'py, PyAny> = - host_entry(py, UserCodeHost::BuildHookIdentities)?.call1((hook_functions,))?; - for item in mapping_items(&identities)?.try_iter()? { - let (name, identity): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; - hashes.set_item(name, native_field(&identity, "version_hash")?)?; - } - Ok(hashes) -} - /// Current direct model identities from code and upstream identities. #[pyfunction] #[pyo3(signature = (**options))] @@ -382,41 +365,9 @@ pub(crate) fn with_honest_model_write_hashes<'py>( replace_with(&changes, ("models", model_changes.into_any())) } -/// Resolved actions with recomputed model changes patched in where available. -#[pyfunction] -#[pyo3(signature = (*, resolved_actions, changes))] -pub(crate) fn merge_recomputed_model_changes<'py>( - resolved_actions: &Bound<'py, PyAny>, - changes: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = resolved_actions.py(); - let recomputed: Bound<'py, PyAny> = native_field(changes, "models")?; - let merged: Bound<'py, PyDict> = PyDict::new(py); - for item in native_field(resolved_actions, "models")? - .call_method0("items")? - .try_iter()? - { - let (name, resolved): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; - let mut change: Bound<'py, PyAny> = mapping_get(&recomputed, &name)?; - if change.is_none() { - change = native_field(&resolved, "change")?; - } - merged.set_item(name, replace_with(&resolved, ("change", change))?)?; - } - replace_with(resolved_actions, ("models", merged.into_any())) -} - -fn replace_with<'py>( +pub(crate) fn replace_with<'py>( instance: &Bound<'py, PyAny>, (field, value): (&str, Bound<'py, PyAny>), ) -> PyResult> { replaced(instance, &[(field, value)]) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - build_direct_model_version_identities, - module - )?)?; - module.add_function(wrap_pyfunction!(merge_recomputed_model_changes, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs similarity index 71% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs rename to crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs index f9a3ca7624..21bbcc1793 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs @@ -1,22 +1,22 @@ //! Version identity metadata JSON and hashes of models, functions and seeds, byte-identical to the recorded fingerprints. +use crate::planner_identity::main::compiled_function_fingerprint_sql::compiled_function_fingerprint_sql; +use crate::planner_identity::main::model_declared_columns_hash::model_declared_columns_hash; +use crate::planner_identity::main::or_empty::or_empty; use std::collections::HashMap; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_identity::constants::{ CONTRACT_CONFIG_KEY, CURSOR_INPUTS_CONFIG_KEY, ENFORCED_CONTRACT_POLICY, MODEL_CUSTOM_CONFIG_KEY, MODEL_PLACEHOLDERS_CONFIG_KEY, MODEL_POST_HOOKS_CONFIG_KEY, - MODEL_PRE_HOOKS_CONFIG_KEY, SQL_HOOK_IDENTITY_FIELDS, TABLE_FN_RESOURCE, UDF_RESOURCE, - VERSION_IDENTITY_CONFIG_KEYS, + MODEL_PRE_HOOKS_CONFIG_KEY, SQL_HOOK_IDENTITY_FIELDS, VERSION_IDENTITY_CONFIG_KEYS, }; -use sqlbuild_analysis::planner_identity::main::function_fingerprint_sql::function_fingerprint_sql; -use sqlbuild_analysis::planner_identity::models::FunctionFingerprintFields; use sqlbuild_core::fingerprint_identity::main::stable_identity_hash::stable_identity_hash; -use crate::bindings::_helpers::microbatches::conversion::timestamp; +use crate::microbatches::main::timestamp::timestamp; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; @@ -27,11 +27,6 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_front::discovery::main::headers::hook_entries::hook_entries; -use sqlbuild_python_front::model_config::main::config::config_value::config_value; -use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::compiled_models::config::config; -use sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps; -use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::planner_identity::main::baselined_function_definition_hash::baselined_function_definition_hash; use sqlbuild_python_project::planner_identity::main::function_definition_hash::function_definition_hash; use sqlbuild_python_project::resource_views::main::function_reads::arguments::arguments; @@ -69,17 +64,6 @@ fn version_identity_config<'py>(config_values: &Bound<'py, PyAny>) -> PyResult( - value: Option>, - py: Python<'py>, -) -> PyResult> { - match value { - Some(value) if value.is_truthy()? => Ok(value), - _ => Ok(PyDict::new(py).into_any()), - } -} - /// Deterministic metadata JSON for model version identity. pub(crate) fn version_identity_metadata_json( model_name: &Bound<'_, PyAny>, @@ -101,44 +85,7 @@ pub(crate) fn version_identity_metadata_json( sorted_dumps(payload.as_any(), COMPACT, true) } -/// Deterministic non-query model fingerprint metadata JSON. -pub(crate) fn model_version_identity_metadata_json<'py>( - model: &Bound<'py, PyAny>, - function_local_hashes: Option<&Bound<'py, PyAny>>, - hook_version_hashes: Option<&Bound<'py, PyAny>>, -) -> PyResult { - let py: Python<'py> = model.py(); - let local_function_hashes: Bound<'py, PyDict> = PyDict::new(py); - if let Some(function_hashes) = function_local_hashes.filter(|hashes| !hashes.is_none()) { - for upstream_key in deps(model)?.try_iter()? { - let upstream_key: Bound<'py, PyAny> = upstream_key?; - let kind: String = native_field(&upstream_key, "resource_type")? - .str()? - .extract()?; - if kind != UDF_RESOURCE && kind != TABLE_FN_RESOURCE { - continue; - } - let name: Bound<'py, PyAny> = native_field(&upstream_key, "name")?; - let upstream_hash: Bound<'py, PyAny> = mapping_get(function_hashes, &name)?; - if !upstream_hash.is_none() { - local_function_hashes.set_item(name, upstream_hash)?; - } - } - } - let config_values: Bound<'py, PyAny> = native_field(&config(model)?, "values")?; - let signature: Bound<'py, PyDict> = - execution_signature(model, &config_values, hook_version_hashes)?; - version_identity_metadata_json( - &name_object(model)?, - &config_values, - ( - Some(local_function_hashes.into_any()), - Some(signature.into_any()), - ), - ) -} - -fn execution_signature<'py>( +pub(crate) fn execution_signature<'py>( model: &Bound<'py, PyAny>, config_values: &Bound<'py, PyAny>, hook_version_hashes: Option<&Bound<'py, PyAny>>, @@ -242,7 +189,7 @@ fn contract_output_signature<'py>( Ok(Some(signature)) } -fn column_signature<'py>(column: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn column_signature<'py>(column: &Bound<'py, PyAny>) -> PyResult> { let entry: Bound<'py, PyDict> = PyDict::new(column.py()); for field in ["name", "type", "nullable"] { entry.set_item(field, native_field(column, field)?)?; @@ -250,7 +197,7 @@ fn column_signature<'py>(column: &Bound<'py, PyAny>) -> PyResult(families: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn family_signatures<'py>(families: &Bound<'py, PyAny>) -> PyResult> { let rendered: Bound<'py, PyList> = PyList::empty(families.py()); for family in families.try_iter()? { let family: Bound<'py, PyAny> = family?; @@ -270,38 +217,6 @@ fn family_signatures<'py>(families: &Bound<'py, PyAny>) -> PyResult) -> PyResult { - let py: Python<'_> = model.py(); - let schema_entry: Bound<'_, PyAny> = - sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( - model, - )?; - let payload: Bound<'_, PyDict> = PyDict::new(py); - payload.set_item( - "contract", - config_value(py, &model_values(model)?, CONTRACT_CONFIG_KEY)?, - )?; - let columns: Bound<'_, PyList> = PyList::empty(py); - let mut families: Bound<'_, PyList> = PyList::empty(py); - let mut type_enforcement: Bound<'_, PyAny> = py.None().into_bound(py); - if !schema_entry.is_none() { - for column in native_field(&schema_entry, "columns")?.try_iter()? { - columns.append(column_signature(&column?)?)?; - } - families = family_signatures(&native_field(&schema_entry, "dynamic_columns")?)?; - type_enforcement = native_field(&schema_entry, "type_enforcement")?; - } - payload.set_item("columns", columns)?; - payload.set_item("dynamic_columns", families)?; - payload.set_item("type_enforcement", type_enforcement)?; - Ok(stable_identity_hash(&sorted_dumps( - payload.as_any(), - COMPACT, - true, - )?)) -} - /// Whether the function's recorded fingerprint came from an older scheme for the same definition. fn recorded_function_unchanged( function: &Bound<'_, PyAny>, @@ -438,7 +353,7 @@ fn text_or_empty(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { } /// `f"{value or ''}"` of an attribute read, empty when the read failed. -fn text_of(read: PyResult>) -> PyResult { +pub(crate) fn text_of(read: PyResult>) -> PyResult { let value: Bound<'_, PyAny> = match read { Ok(value) => value, Err(_) => return Ok(String::new()), @@ -450,7 +365,7 @@ fn text_of(read: PyResult>) -> PyResult { } } -fn rendered_pairs(items: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn rendered_pairs(items: &Bound<'_, PyAny>) -> PyResult> { items .try_iter()? .map(|item| { @@ -460,28 +375,6 @@ fn rendered_pairs(items: &Bound<'_, PyAny>) -> PyResult> { .collect() } -/// Stable function definition text used for change fingerprints. -pub(crate) fn compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { - Ok(function_fingerprint_sql(&FunctionFingerprintFields { - language: - sqlbuild_python_project::resource_views::main::function_reads::language::language( - function, - )? - .str()? - .extract()?, - arguments: rendered_pairs(&arguments(function)?)?, - returns: returns(function)?.str()?.extract()?, - return_columns: rendered_pairs(&return_columns(function)?)?, - runtime_version: text_of(runtime_version(function))?, - entry_point: text_of(entry_point(function))?, - packages: packages(function)? - .try_iter()? - .map(|package| package?.extract::()) - .collect::>()?, - body_sql: body_sql(function)?.extract()?, - })) -} - /// `(identity_hash, metadata_json)` of one compiled seed from its content and load-affecting config. pub(crate) fn seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { let py: Python<'_> = seed.py(); @@ -561,7 +454,7 @@ fn seed_rows<'py>( /// Deterministic metadata JSON for model version identity. #[pyfunction] #[pyo3(signature = (*, model_name, config_values, local_function_hashes = None, execution_signature = None))] -fn build_version_identity_metadata_json<'py>( +pub(crate) fn build_version_identity_metadata_json<'py>( model_name: &Bound<'py, PyAny>, config_values: &Bound<'py, PyAny>, local_function_hashes: Option>, @@ -574,70 +467,9 @@ fn build_version_identity_metadata_json<'py>( ) } -/// Deterministic non-query model fingerprint metadata JSON. -#[pyfunction] -#[pyo3(signature = (*, model, function_local_hashes = None, hook_version_hashes = None))] -pub(crate) fn build_model_version_identity_metadata_json<'py>( - model: &Bound<'py, PyAny>, - function_local_hashes: Option>, - hook_version_hashes: Option>, -) -> PyResult { - model_version_identity_metadata_json( - model, - function_local_hashes.as_ref(), - hook_version_hashes.as_ref(), - ) -} - /// Hash the model's own declared columns, column families and contract setting. #[pyfunction] #[pyo3(signature = (*, model))] -fn declared_columns_hash(model: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn declared_columns_hash(model: &Bound<'_, PyAny>) -> PyResult { model_declared_columns_hash(model) } - -/// Local-only semantic hashes of functions by name. -#[pyfunction] -#[pyo3(signature = (*, functions, dialect))] -pub(crate) fn build_function_local_hashes<'py>( - functions: &Bound<'py, PyAny>, - dialect: Option<&str>, -) -> PyResult> { - function_local_hashes( - functions, - dialect, - &functions.py().None().into_bound(functions.py()), - ) -} - -/// Stable function definition text used for change fingerprints. -#[pyfunction] -pub(crate) fn build_compiled_function_fingerprint_sql( - function: &Bound<'_, PyAny>, -) -> PyResult { - compiled_function_fingerprint_sql(function) -} - -/// `(identity_hash, metadata_json)` for one compiled seed. -#[pyfunction] -pub(crate) fn build_seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { - seed_identity(seed) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - build_version_identity_metadata_json, - module - )?)?; - module.add_function(wrap_pyfunction!( - build_model_version_identity_metadata_json, - module - )?)?; - module.add_function(wrap_pyfunction!(declared_columns_hash, module)?)?; - module.add_function(wrap_pyfunction!(build_function_local_hashes, module)?)?; - module.add_function(wrap_pyfunction!( - build_compiled_function_fingerprint_sql, - module - )?)?; - module.add_function(wrap_pyfunction!(build_seed_identity, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/mod.rs new file mode 100644 index 0000000000..29511a7cd3 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod identities; +pub(crate) mod metadata; +pub(crate) mod staleness; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs similarity index 63% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs rename to crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs index bbdd4ec732..1dc365c694 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs @@ -1,24 +1,21 @@ //! Warnings for selected models stale through changed upstreams outside the run set. -use crate::bindings::_helpers::planner_changes::results::native_enum; +use crate::planner_changes::main::native_enum::native_enum; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use std::collections::{HashMap, HashSet}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyString, PyTuple}; -use sqlbuild_analysis::planner_identity::constants::{ - MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE, -}; use sqlbuild_analysis::planner_identity::main::classify_selection_staleness::classify_selection_staleness; use sqlbuild_analysis::planner_identity::main::stale_upstream_warning_message::stale_upstream_warning_message; use sqlbuild_analysis::planner_identity::models::{ GraphKey, StaleWarning, StalenessGraph, StalenessNames, }; -use crate::bindings::_helpers::planner_changes::detect::{ModelPass, model_change}; -use sqlbuild_python_base::boundary::models::KeywordArguments; +use crate::planner_changes::main::model_change::model_change; +use crate::planner_changes::models::ModelPass; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; @@ -54,15 +51,15 @@ fn upstream_graph(upstream_deps: &Bound<'_, PyAny>) -> PyResult { - original_scope: Bound<'py, PyAny>, - changes: Bound<'py, PyAny>, - snapshot: Bound<'py, PyAny>, - identities: Bound<'py, PyAny>, - model_changes_complete: bool, +pub(crate) struct StalenessInputs<'py> { + pub(crate) original_scope: Bound<'py, PyAny>, + pub(crate) changes: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) identities: Bound<'py, PyAny>, + pub(crate) model_changes_complete: bool, } -fn changed_model_names(inputs: &StalenessInputs<'_>) -> PyResult> { +pub(crate) fn changed_model_names(inputs: &StalenessInputs<'_>) -> PyResult> { let py: Python<'_> = inputs.snapshot.py(); let models: Bound<'_, PyAny> = native_field(&inputs.changes, "models")?; let mut changed: HashSet = HashSet::new(); @@ -113,7 +110,7 @@ fn changed_model_names(inputs: &StalenessInputs<'_>) -> PyResult Ok(changed) } -fn changed_seed_names(inputs: &StalenessInputs<'_>) -> PyResult> { +pub(crate) fn changed_seed_names(inputs: &StalenessInputs<'_>) -> PyResult> { let recorded: Bound<'_, PyAny> = native_field(&native_field(&inputs.snapshot, "fingerprints")?, "seeds")?; let mut changed: HashSet = HashSet::new(); @@ -133,7 +130,7 @@ fn changed_seed_names(inputs: &StalenessInputs<'_>) -> PyResult> Ok(changed) } -fn plan_warnings<'py>( +pub(crate) fn plan_warnings<'py>( py: Python<'py>, warnings: Vec, ) -> PyResult> { @@ -161,92 +158,13 @@ fn plan_warnings<'py>( PyTuple::new(py, rendered) } -/// Warn for selected models stale through changed upstreams outside the run set. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn build_stale_out_of_selection_warnings<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "build_stale_out_of_selection_warnings", - options, - ( - &[ - "original_scope", - "execution_scope", - "changes", - "snapshot", - "version_identities", - "source_freshness", - ], - &["include_sources", "model_changes_complete"], - ), - )?; - let include_sources: bool = - !arguments.passed("include_sources")? || arguments.flag("include_sources")?; - let inputs = StalenessInputs { - original_scope: arguments.value("original_scope")?, - changes: arguments.value("changes")?, - snapshot: arguments.value("snapshot")?, - identities: arguments.value("version_identities")?, - model_changes_complete: arguments.flag("model_changes_complete")?, - }; - let run = scope_data(&arguments.value("execution_scope")?)?; - let original = scope_data(&inputs.original_scope)?; - let names = |kind: &str| -> HashSet { - run.selected - .iter() - .filter(|key| key.0 == kind) - .map(|key| key.1.clone()) - .collect() - }; - let source_freshness: Bound<'py, PyAny> = arguments.value("source_freshness")?; - let mut changed_sources: HashSet = HashSet::new(); - if include_sources && !source_freshness.is_none() { - for identity in native_field(&source_freshness, "changed_identities")?.try_iter()? { - changed_sources.insert(native_field(&identity?, "source_name")?.extract()?); - } - } - let graph = StalenessGraph { - upstream_deps: original - .upstream - .iter() - .map(|(key, deps)| { - ( - GraphKey::from(key), - deps.iter().map(GraphKey::from).collect(), - ) - }) - .collect(), - selected_model_names: original - .selected - .iter() - .filter(|key| key.0 == MODEL_RESOURCE) - .map(|key| key.1.clone()) - .collect(), - run: StalenessNames { - models: names(MODEL_RESOURCE), - seeds: names(SEED_RESOURCE), - sources: names(SOURCE_RESOURCE), - }, - changed: StalenessNames { - models: changed_model_names(&inputs)?, - seeds: changed_seed_names(&inputs)?, - sources: changed_sources, - }, - }; - plan_warnings(py, classify_selection_staleness(&graph)) -} - fn name_set(values: &Bound<'_, PyAny>) -> PyResult> { values.try_iter()?.map(|value| value?.extract()).collect() } /// Stale warnings for selected models with changed upstreams outside the run set. #[pyfunction] -fn classify_selection_staleness_warnings<'py>( +pub(crate) fn classify_selection_staleness_warnings<'py>( graph: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = graph.py(); @@ -285,14 +203,3 @@ fn classify_selection_staleness_warnings<'py>( } PyTuple::new(py, warnings) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - build_stale_out_of_selection_warnings, - module - )?)?; - module.add_function(wrap_pyfunction!( - classify_selection_staleness_warnings, - module - )?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/build_compiled_function_fingerprint_sql.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/build_compiled_function_fingerprint_sql.rs new file mode 100644 index 0000000000..2cb92f048e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/build_compiled_function_fingerprint_sql.rs @@ -0,0 +1,11 @@ +//! Stable function definition text used for change fingerprints. + +use crate::planner_identity::main::compiled_function_fingerprint_sql::compiled_function_fingerprint_sql; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; + +/// Stable function definition text used for change fingerprints. +#[pyfunction] +pub fn build_compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { + compiled_function_fingerprint_sql(function) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/build_function_local_hashes.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/build_function_local_hashes.rs new file mode 100644 index 0000000000..340537accd --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/build_function_local_hashes.rs @@ -0,0 +1,20 @@ +//! Local-only semantic hashes of functions by name. + +use crate::planner_identity::_helpers::metadata::function_local_hashes; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyDict; + +/// Local-only semantic hashes of functions by name. +#[pyfunction] +#[pyo3(signature = (*, functions, dialect))] +pub fn build_function_local_hashes<'py>( + functions: &Bound<'py, PyAny>, + dialect: Option<&str>, +) -> PyResult> { + function_local_hashes( + functions, + dialect, + &functions.py().None().into_bound(functions.py()), + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/build_model_version_identity_metadata_json.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/build_model_version_identity_metadata_json.rs new file mode 100644 index 0000000000..8e3213981e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/build_model_version_identity_metadata_json.rs @@ -0,0 +1,20 @@ +//! Deterministic non-query model fingerprint metadata JSON. + +use crate::planner_identity::main::model_version_identity_metadata_json::model_version_identity_metadata_json; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; + +/// Deterministic non-query model fingerprint metadata JSON. +#[pyfunction] +#[pyo3(signature = (*, model, function_local_hashes = None, hook_version_hashes = None))] +pub fn build_model_version_identity_metadata_json<'py>( + model: &Bound<'py, PyAny>, + function_local_hashes: Option>, + hook_version_hashes: Option>, +) -> PyResult { + model_version_identity_metadata_json( + model, + function_local_hashes.as_ref(), + hook_version_hashes.as_ref(), + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/build_seed_identity.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/build_seed_identity.rs new file mode 100644 index 0000000000..e0205428ad --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/build_seed_identity.rs @@ -0,0 +1,11 @@ +//! `(identity_hash, metadata_json)` for one compiled seed. + +use crate::planner_identity::_helpers::metadata::seed_identity; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; + +/// `(identity_hash, metadata_json)` for one compiled seed. +#[pyfunction] +pub fn build_seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + seed_identity(seed) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/build_stale_out_of_selection_warnings.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/build_stale_out_of_selection_warnings.rs new file mode 100644 index 0000000000..67ea2a50db --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/build_stale_out_of_selection_warnings.rs @@ -0,0 +1,97 @@ +//! Warn for selected models stale through changed upstreams outside the run set. + +use crate::planner_identity::_helpers::staleness::StalenessInputs; +use crate::planner_identity::_helpers::staleness::changed_model_names; +use crate::planner_identity::_helpers::staleness::changed_seed_names; +use crate::planner_identity::_helpers::staleness::plan_warnings; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_analysis::planner_identity::constants::{ + MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE, +}; +use sqlbuild_analysis::planner_identity::main::classify_selection_staleness::classify_selection_staleness; +use sqlbuild_analysis::planner_identity::models::{GraphKey, StalenessGraph, StalenessNames}; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use std::collections::HashSet; + +/// Warn for selected models stale through changed upstreams outside the run set. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn build_stale_out_of_selection_warnings<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_stale_out_of_selection_warnings", + options, + ( + &[ + "original_scope", + "execution_scope", + "changes", + "snapshot", + "version_identities", + "source_freshness", + ], + &["include_sources", "model_changes_complete"], + ), + )?; + let include_sources: bool = + !arguments.passed("include_sources")? || arguments.flag("include_sources")?; + let inputs = StalenessInputs { + original_scope: arguments.value("original_scope")?, + changes: arguments.value("changes")?, + snapshot: arguments.value("snapshot")?, + identities: arguments.value("version_identities")?, + model_changes_complete: arguments.flag("model_changes_complete")?, + }; + let run = scope_data(&arguments.value("execution_scope")?)?; + let original = scope_data(&inputs.original_scope)?; + let names = |kind: &str| -> HashSet { + run.selected + .iter() + .filter(|key| key.0 == kind) + .map(|key| key.1.clone()) + .collect() + }; + let source_freshness: Bound<'py, PyAny> = arguments.value("source_freshness")?; + let mut changed_sources: HashSet = HashSet::new(); + if include_sources && !source_freshness.is_none() { + for identity in native_field(&source_freshness, "changed_identities")?.try_iter()? { + changed_sources.insert(native_field(&identity?, "source_name")?.extract()?); + } + } + let graph = StalenessGraph { + upstream_deps: original + .upstream + .iter() + .map(|(key, deps)| { + ( + GraphKey::from(key), + deps.iter().map(GraphKey::from).collect(), + ) + }) + .collect(), + selected_model_names: original + .selected + .iter() + .filter(|key| key.0 == MODEL_RESOURCE) + .map(|key| key.1.clone()) + .collect(), + run: StalenessNames { + models: names(MODEL_RESOURCE), + seeds: names(SEED_RESOURCE), + sources: names(SOURCE_RESOURCE), + }, + changed: StalenessNames { + models: changed_model_names(&inputs)?, + seeds: changed_seed_names(&inputs)?, + sources: changed_sources, + }, + }; + plan_warnings(py, classify_selection_staleness(&graph)) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/compiled_function_fingerprint_sql.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/compiled_function_fingerprint_sql.rs new file mode 100644 index 0000000000..14ddb5ab0f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/compiled_function_fingerprint_sql.rs @@ -0,0 +1,36 @@ +//! Stable function definition text used for change fingerprints. + +use crate::planner_identity::_helpers::metadata::rendered_pairs; +use crate::planner_identity::_helpers::metadata::text_of; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::planner_identity::main::function_fingerprint_sql::function_fingerprint_sql; +use sqlbuild_analysis::planner_identity::models::FunctionFingerprintFields; +use sqlbuild_python_project::resource_views::main::function_reads::arguments::arguments; +use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; +use sqlbuild_python_project::resource_views::main::function_reads::entry_point::entry_point; +use sqlbuild_python_project::resource_views::main::function_reads::packages::packages; +use sqlbuild_python_project::resource_views::main::function_reads::return_columns::return_columns; +use sqlbuild_python_project::resource_views::main::function_reads::returns::returns; +use sqlbuild_python_project::resource_views::main::function_reads::runtime_version::runtime_version; + +/// Stable function definition text used for change fingerprints. +pub fn compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { + Ok(function_fingerprint_sql(&FunctionFingerprintFields { + language: + sqlbuild_python_project::resource_views::main::function_reads::language::language( + function, + )? + .str()? + .extract()?, + arguments: rendered_pairs(&arguments(function)?)?, + returns: returns(function)?.str()?.extract()?, + return_columns: rendered_pairs(&return_columns(function)?)?, + runtime_version: text_of(runtime_version(function))?, + entry_point: text_of(entry_point(function))?, + packages: packages(function)? + .try_iter()? + .map(|package| package?.extract::()) + .collect::>()?, + body_sql: body_sql(function)?.extract()?, + })) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/direct_model_version_identities.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/direct_model_version_identities.rs new file mode 100644 index 0000000000..2a4e65dd89 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/direct_model_version_identities.rs @@ -0,0 +1,20 @@ +//! Direct model identities of `(functions, scope)`; unchanged nodes keep their `recorded` fingerprints. + +use crate::planner_identity::_helpers::identities; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Direct model identities of `(functions, scope)`; unchanged nodes keep their `recorded` fingerprints. +#[inline] +pub fn direct_model_version_identities<'py>( + (functions, scope): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + dialect: Option, + (seeds, source_hashes, hooks): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + recorded: &Bound<'py, PyAny>, +) -> PyResult> { + identities::direct_model_version_identities( + (functions, scope), + dialect, + (seeds, source_hashes, hooks), + recorded, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/function_local_hashes.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/function_local_hashes.rs new file mode 100644 index 0000000000..b4c9e4267c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/function_local_hashes.rs @@ -0,0 +1,15 @@ +//! Local hashes of functions by name; unchanged functions keep the latest local hash `recorded` models saw. + +use crate::planner_identity::_helpers::metadata; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyDict; + +/// Local hashes of functions by name; unchanged functions keep the latest local hash `recorded` models saw. +#[inline] +pub fn function_local_hashes<'py>( + functions: &Bound<'py, PyAny>, + dialect: Option<&str>, + recorded: &Bound<'py, PyAny>, +) -> PyResult> { + metadata::function_local_hashes(functions, dialect, recorded) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/hook_version_hashes.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/hook_version_hashes.rs new file mode 100644 index 0000000000..d8ee24c528 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/hook_version_hashes.rs @@ -0,0 +1,23 @@ +//! `{name: identity.version_hash}` of the discovered Python hooks. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; + +/// `{name: identity.version_hash}` of the discovered Python hooks. +pub fn hook_version_hashes<'py>( + hook_functions: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = hook_functions.py(); + let hashes: Bound<'py, PyDict> = PyDict::new(py); + let identities: Bound<'py, PyAny> = + host_entry(py, UserCodeHost::BuildHookIdentities)?.call1((hook_functions,))?; + for item in mapping_items(&identities)?.try_iter()? { + let (name, identity): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + hashes.set_item(name, native_field(&identity, "version_hash")?)?; + } + Ok(hashes) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/merge_recomputed_model_changes.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/merge_recomputed_model_changes.rs new file mode 100644 index 0000000000..2cd53509c8 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/merge_recomputed_model_changes.rs @@ -0,0 +1,32 @@ +//! Resolved actions with recomputed model changes patched in where available. + +use crate::planner_identity::_helpers::identities::replace_with; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; + +/// Resolved actions with recomputed model changes patched in where available. +#[pyfunction] +#[pyo3(signature = (*, resolved_actions, changes))] +pub fn merge_recomputed_model_changes<'py>( + resolved_actions: &Bound<'py, PyAny>, + changes: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = resolved_actions.py(); + let recomputed: Bound<'py, PyAny> = native_field(changes, "models")?; + let merged: Bound<'py, PyDict> = PyDict::new(py); + for item in native_field(resolved_actions, "models")? + .call_method0("items")? + .try_iter()? + { + let (name, resolved): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let mut change: Bound<'py, PyAny> = mapping_get(&recomputed, &name)?; + if change.is_none() { + change = native_field(&resolved, "change")?; + } + merged.set_item(name, replace_with(&resolved, ("change", change))?)?; + } + replace_with(resolved_actions, ("models", merged.into_any())) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/mod.rs new file mode 100644 index 0000000000..37ba3e304e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/mod.rs @@ -0,0 +1,18 @@ +pub mod build_compiled_function_fingerprint_sql; +pub mod build_function_local_hashes; +pub mod build_model_version_identity_metadata_json; +pub mod build_seed_identity; +pub mod build_stale_out_of_selection_warnings; +pub mod compiled_function_fingerprint_sql; +pub mod direct_model_version_identities; +pub mod function_local_hashes; +pub mod hook_version_hashes; +pub mod merge_recomputed_model_changes; +pub mod model_declared_columns_hash; +pub mod model_version_identity_metadata_json; +pub mod or_empty; +pub mod register_identities; +pub mod register_metadata; +pub mod register_staleness; +pub mod seed_identity; +pub mod with_honest_model_write_hashes; diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/model_declared_columns_hash.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/model_declared_columns_hash.rs new file mode 100644 index 0000000000..f957eaadd1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/model_declared_columns_hash.rs @@ -0,0 +1,45 @@ +//! The hash of a model's own declared columns, column families and contract setting. + +use crate::planner_identity::_helpers::metadata::column_signature; +use crate::planner_identity::_helpers::metadata::family_signatures; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods}; +use sqlbuild_analysis::planner_identity::constants::CONTRACT_CONFIG_KEY; +use sqlbuild_core::fingerprint_identity::main::stable_identity_hash::stable_identity_hash; +use sqlbuild_python_base::planner_identity::constants::COMPACT; +use sqlbuild_python_base::planner_identity::main::sorted_dumps::sorted_dumps; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; + +/// The hash of a model's own declared columns, column families and contract setting. +pub fn model_declared_columns_hash(model: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = model.py(); + let schema_entry: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( + model, + )?; + let payload: Bound<'_, PyDict> = PyDict::new(py); + payload.set_item( + "contract", + config_value(py, &model_values(model)?, CONTRACT_CONFIG_KEY)?, + )?; + let columns: Bound<'_, PyList> = PyList::empty(py); + let mut families: Bound<'_, PyList> = PyList::empty(py); + let mut type_enforcement: Bound<'_, PyAny> = py.None().into_bound(py); + if !schema_entry.is_none() { + for column in native_field(&schema_entry, "columns")?.try_iter()? { + columns.append(column_signature(&column?)?)?; + } + families = family_signatures(&native_field(&schema_entry, "dynamic_columns")?)?; + type_enforcement = native_field(&schema_entry, "type_enforcement")?; + } + payload.set_item("columns", columns)?; + payload.set_item("dynamic_columns", families)?; + payload.set_item("type_enforcement", type_enforcement)?; + Ok(stable_identity_hash(&sorted_dumps( + payload.as_any(), + COMPACT, + true, + )?)) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/model_version_identity_metadata_json.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/model_version_identity_metadata_json.rs new file mode 100644 index 0000000000..e36be6e0d3 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/model_version_identity_metadata_json.rs @@ -0,0 +1,49 @@ +//! Deterministic non-query model fingerprint metadata JSON. + +use crate::planner_identity::_helpers::metadata::execution_signature; +use crate::planner_identity::_helpers::metadata::version_identity_metadata_json; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::planner_identity::constants::{TABLE_FN_RESOURCE, UDF_RESOURCE}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_project::compiled_views::main::compiled_models::config::config; +use sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; + +/// Deterministic non-query model fingerprint metadata JSON. +pub fn model_version_identity_metadata_json<'py>( + model: &Bound<'py, PyAny>, + function_local_hashes: Option<&Bound<'py, PyAny>>, + hook_version_hashes: Option<&Bound<'py, PyAny>>, +) -> PyResult { + let py: Python<'py> = model.py(); + let local_function_hashes: Bound<'py, PyDict> = PyDict::new(py); + if let Some(function_hashes) = function_local_hashes.filter(|hashes| !hashes.is_none()) { + for upstream_key in deps(model)?.try_iter()? { + let upstream_key: Bound<'py, PyAny> = upstream_key?; + let kind: String = native_field(&upstream_key, "resource_type")? + .str()? + .extract()?; + if kind != UDF_RESOURCE && kind != TABLE_FN_RESOURCE { + continue; + } + let name: Bound<'py, PyAny> = native_field(&upstream_key, "name")?; + let upstream_hash: Bound<'py, PyAny> = mapping_get(function_hashes, &name)?; + if !upstream_hash.is_none() { + local_function_hashes.set_item(name, upstream_hash)?; + } + } + } + let config_values: Bound<'py, PyAny> = native_field(&config(model)?, "values")?; + let signature: Bound<'py, PyDict> = + execution_signature(model, &config_values, hook_version_hashes)?; + version_identity_metadata_json( + &name_object(model)?, + &config_values, + ( + Some(local_function_hashes.into_any()), + Some(signature.into_any()), + ), + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/or_empty.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/or_empty.rs new file mode 100644 index 0000000000..46bce02f05 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/or_empty.rs @@ -0,0 +1,15 @@ +//! `value or {}`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; + +/// `value or {}`. +pub fn or_empty<'py>( + value: Option>, + py: Python<'py>, +) -> PyResult> { + match value { + Some(value) if value.is_truthy()? => Ok(value), + _ => Ok(PyDict::new(py).into_any()), + } +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/register_identities.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/register_identities.rs new file mode 100644 index 0000000000..326afbe429 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/register_identities.rs @@ -0,0 +1,15 @@ +//! Register the identities functions and classes. + +use crate::planner_identity::_helpers::identities::build_direct_model_version_identities; +use crate::planner_identity::main::merge_recomputed_model_changes::merge_recomputed_model_changes; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the identities functions and classes. +pub fn register_identities(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_direct_model_version_identities, + module + )?)?; + module.add_function(wrap_pyfunction!(merge_recomputed_model_changes, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/register_metadata.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/register_metadata.rs new file mode 100644 index 0000000000..427c149ff5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/register_metadata.rs @@ -0,0 +1,29 @@ +//! Register the metadata functions and classes. + +use crate::planner_identity::_helpers::metadata::build_version_identity_metadata_json; +use crate::planner_identity::_helpers::metadata::declared_columns_hash; +use crate::planner_identity::main::build_compiled_function_fingerprint_sql::build_compiled_function_fingerprint_sql; +use crate::planner_identity::main::build_function_local_hashes::build_function_local_hashes; +use crate::planner_identity::main::build_model_version_identity_metadata_json::build_model_version_identity_metadata_json; +use crate::planner_identity::main::build_seed_identity::build_seed_identity; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the metadata functions and classes. +pub fn register_metadata(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_version_identity_metadata_json, + module + )?)?; + module.add_function(wrap_pyfunction!( + build_model_version_identity_metadata_json, + module + )?)?; + module.add_function(wrap_pyfunction!(declared_columns_hash, module)?)?; + module.add_function(wrap_pyfunction!(build_function_local_hashes, module)?)?; + module.add_function(wrap_pyfunction!( + build_compiled_function_fingerprint_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(build_seed_identity, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/register_staleness.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/register_staleness.rs new file mode 100644 index 0000000000..f65f88c5f7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/register_staleness.rs @@ -0,0 +1,18 @@ +//! Register the staleness functions and classes. + +use crate::planner_identity::_helpers::staleness::classify_selection_staleness_warnings; +use crate::planner_identity::main::build_stale_out_of_selection_warnings::build_stale_out_of_selection_warnings; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the staleness functions and classes. +pub fn register_staleness(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_stale_out_of_selection_warnings, + module + )?)?; + module.add_function(wrap_pyfunction!( + classify_selection_staleness_warnings, + module + )?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/seed_identity.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/seed_identity.rs new file mode 100644 index 0000000000..b6d5ffa984 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/seed_identity.rs @@ -0,0 +1,10 @@ +//! `(identity_hash, metadata_json)` of one compiled seed from its content and load-affecting config. + +use crate::planner_identity::_helpers::metadata; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// `(identity_hash, metadata_json)` of one compiled seed from its content and load-affecting config. +#[inline] +pub fn seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + metadata::seed_identity(seed) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/with_honest_model_write_hashes.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/with_honest_model_write_hashes.rs new file mode 100644 index 0000000000..4f396f8310 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/with_honest_model_write_hashes.rs @@ -0,0 +1,14 @@ +//! Planned write hashes replaced by hashes based on upstreams available this run. + +use crate::planner_identity::_helpers::identities; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyDict; + +/// Planned write hashes replaced by hashes based on upstreams available this run. +#[inline] +pub fn with_honest_model_write_hashes<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + identities::with_honest_model_write_hashes(py, options) +} diff --git a/crates/sqlbuild-python-planner/src/planner_identity/mod.rs b/crates/sqlbuild-python-planner/src/planner_identity/mod.rs new file mode 100644 index 0000000000..cc3f2ebbb6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_identity/mod.rs @@ -0,0 +1,4 @@ +//! Planner version identities, their metadata and hashes, and staleness warnings. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs index 6f8df1dd2e..8e0c01fe50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs @@ -1,12 +1,8 @@ //! Gather live candidate evidence without changing adapter inspection ownership. -use crate::bindings::_helpers::planner_migrations::column_objects::{ - config, definition_hash, lower, -}; -use crate::bindings::_helpers::planner_migrations::models::{ - ColumnCandidate, ColumnMigrationInputs, MigrationPhase, -}; -use crate::bindings::_helpers::planner_migrations::projections::parse_query_shape; +use crate::planner_migrations::_helpers::column::column_objects::{config, definition_hash, lower}; +use crate::planner_migrations::_helpers::projections::query_shapes::parse_query_shape; +use crate::planner_migrations::models::{ColumnCandidate, ColumnMigrationInputs, MigrationPhase}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; @@ -16,7 +12,7 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::q use sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; -pub(super) fn candidates<'py>( +pub(crate) fn candidates<'py>( (phase, columns): (&MigrationPhase<'py>, &ColumnMigrationInputs<'py>), dialect: &Bound<'py, PyAny>, ) -> PyResult>> { @@ -99,7 +95,7 @@ pub(super) fn candidates<'py>( Ok(result) } -pub(super) fn declared_names<'py>( +pub(crate) fn declared_names<'py>( candidate: &ColumnCandidate<'py>, ) -> PyResult> { let mut names: Vec = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs index 1ea1e3457b..18b11be9ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs @@ -2,13 +2,13 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::planner_migrations::column_candidates::declared_names; -use crate::bindings::_helpers::planner_migrations::column_objects::{lower, model, replace}; -use crate::bindings::_helpers::planner_migrations::columns::identical_renames; -use crate::bindings::_helpers::planner_migrations::columns::rename_hints; -use crate::bindings::_helpers::planner_migrations::constants::{AUTOMATIC, DONE}; -use crate::bindings::_helpers::planner_migrations::models::{CheckedColumns, ColumnCandidate}; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_migrations::_helpers::column::column_candidates::declared_names; +use crate::planner_migrations::_helpers::column::column_objects::{lower, model, replace}; +use crate::planner_migrations::_helpers::column::columns::identical_renames; +use crate::planner_migrations::_helpers::column::columns::rename_hints; +use crate::planner_migrations::constants::{AUTOMATIC, DONE}; +use crate::planner_migrations::models::{CheckedColumns, ColumnCandidate}; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; use sqlbuild_analysis::planner_migrations::main::column_decision::column_decision; @@ -24,7 +24,7 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_row use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::planner_phases::main::object::object; -pub(super) fn decide<'py>( +pub(crate) fn decide<'py>( runtime: &Bound<'py, PyAny>, candidate: &ColumnCandidate<'py>, events: &Bound<'py, PyAny>, @@ -148,7 +148,7 @@ pub(super) fn decide<'py>( Ok(entries) } -pub(super) fn checked<'py>( +pub(crate) fn checked<'py>( runtime: &Bound<'py, PyAny>, candidate: &ColumnCandidate<'py>, entries: Vec>, @@ -213,10 +213,12 @@ pub(super) fn checked<'py>( ], )?); } else { - warnings.push(super::column_warnings::unavailable( - &entry, - &reason.extract::()?, - )?); + warnings.push( + crate::planner_migrations::_helpers::column::column_warnings::unavailable( + &entry, + &reason.extract::()?, + )?, + ); } } Ok(CheckedColumns { @@ -225,7 +227,7 @@ pub(super) fn checked<'py>( }) } -pub(super) fn hints<'py>( +pub(crate) fn hints<'py>( candidate: &ColumnCandidate<'py>, entries: &[Bound<'py, PyAny>], ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs index 4c39e4bb25..1af84ca8fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs @@ -13,15 +13,15 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::q use sqlbuild_python_project::planner_identity::main::model_definition_hash::model_definition_hash; use sqlbuild_python_project::planner_phases::main::object::object; -pub(super) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { +pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() } -pub(super) fn lower(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn lower(value: &Bound<'_, PyAny>) -> PyResult { value.call_method0("lower")?.extract() } -pub(super) fn replace<'py>( +pub(crate) fn replace<'py>( value: &Bound<'py, PyAny>, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { @@ -32,7 +32,7 @@ pub(super) fn replace<'py>( replaced_keywords(value, &kwargs) } -pub(super) fn model<'py>( +pub(crate) fn model<'py>( py: Python<'py>, name: RecordId, fields: &[(&str, Bound<'py, PyAny>)], @@ -40,7 +40,7 @@ pub(super) fn model<'py>( new_record_with(py, name, fields) } -pub(super) fn native<'py>( +pub(crate) fn native<'py>( py: Python<'py>, name: RecordId, fields: &[(&str, Bound<'py, PyAny>)], @@ -48,13 +48,13 @@ pub(super) fn native<'py>( new_record_with(py, name, fields) } -pub(super) fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { +pub(crate) fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { let py = model.py(); Ok(config_text_object(py, &model_values(model)?, key)? .unwrap_or_else(|| py.None().into_bound(py))) } -pub(super) fn definition_hash<'py>( +pub(crate) fn definition_hash<'py>( model: &Bound<'py, PyAny>, snapshot: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs index f8302572dc..3d9eecefc4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs @@ -1,13 +1,11 @@ //! Project column renames onto the existing snapshot and redirect cursor reads. -use crate::bindings::_helpers::planner_migrations::column_objects::{ +use crate::planner_migrations::_helpers::column::column_objects::{ config, definition_hash, lower, native, replace, }; -use crate::bindings::_helpers::planner_migrations::models::ColumnCandidate; -use crate::bindings::_helpers::planner_migrations::models::{ - ColumnMigrationInputs, MigrationPhase, -}; -use crate::bindings::_helpers::planner_migrations::projections::renames_explain_change; +use crate::planner_migrations::_helpers::projections::query_shapes::renames_explain_change; +use crate::planner_migrations::models::ColumnCandidate; +use crate::planner_migrations::models::{ColumnMigrationInputs, MigrationPhase}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::imported::imported; @@ -22,7 +20,7 @@ use sqlbuild_python_project::native_host::models::ModelLookup; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; -pub(super) fn overlay<'py>( +pub(crate) fn overlay<'py>( snapshot: &Bound<'py, PyAny>, candidates: &[ColumnCandidate<'py>], entries: &[Bound<'py, PyAny>], @@ -129,7 +127,7 @@ pub(super) fn overlay<'py>( ) } -pub(super) fn cursors<'py>( +pub(crate) fn cursors<'py>( (phase, columns): (&MigrationPhase<'py>, &ColumnMigrationInputs<'py>), snapshot: Bound<'py, PyAny>, entries: &[Bound<'py, PyAny>], @@ -173,7 +171,7 @@ pub(super) fn cursors<'py>( }; targets.set_item(name, target)?; } - let redirected = super::planning_boundary::redirected_cursor_snapshots( + let redirected = crate::planner_migrations::_helpers::planning::planning_boundary::redirected_cursor_snapshots( phase, ( native_field(&snapshot, "existing_relations")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs similarity index 70% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs index 77ea8f59e0..32d4707cb3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs @@ -1,14 +1,14 @@ //! Host column migration planning while Python owns warehouse inspection. -use crate::bindings::_helpers::planner_migrations::column_objects::replace; -use crate::bindings::_helpers::planner_migrations::known_input_columns::KnownInputColumns; -use crate::bindings::_helpers::planner_migrations::models::{ +use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_migrations::_helpers::column::known_input_columns::KnownInputColumns; +use crate::planner_migrations::models::{ ColumnMigrationInputs, ColumnMigrationPlanning, MigrationPhase, }; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PySet, PySetMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -19,7 +19,7 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::name_object: /// Decide column renames and return the existing planning result model. #[pyfunction(name = "plan_column_migrations")] #[pyo3(signature = (**options))] -fn plan_column_migrations_of<'py>( +pub(crate) fn plan_column_migrations_of<'py>( py: Python<'py>, options: Option>, ) -> PyResult<()> { @@ -67,7 +67,7 @@ pub(crate) fn plan_column_migrations<'py>( let runtime = phase.runtime.clone(); let snapshot = phase.snapshot.clone(); let adapter = runtime_field(&runtime, |runtime| &runtime.adapter)?; - let candidates = super::column_candidates::candidates( + let candidates = crate::planner_migrations::_helpers::column::column_candidates::candidates( (phase, columns), &adapter.call_method0("sql_analysis_dialect")?, )?; @@ -79,7 +79,9 @@ pub(crate) fn plan_column_migrations<'py>( warnings: Vec::new(), }); } - let state = super::state_inspection::inspection(&runtime, None)?; + let state = crate::planner_migrations::_helpers::discovery::state_inspection::inspection( + &runtime, None, + )?; let schemas = PySet::empty(py)?; for candidate in &candidates { let schema = native_field(&destination(&candidate.model)?, "schema")?; @@ -103,29 +105,50 @@ pub(crate) fn plan_column_migrations<'py>( let mut warnings: Vec> = Vec::new(); let mut hints: Vec> = Vec::new(); for candidate in &candidates { - let decided = super::column_entries::decide( + let decided = crate::planner_migrations::_helpers::column::column_entries::decide( &runtime, candidate, &native_field(&state, "column_events")?, &inputs, )?; let physical = mapping_get(&columns.physical_relations, name_object(&candidate.model)?)?; - let checked = super::column_entries::checked(&runtime, candidate, decided, &physical)?; - for hint in super::column_entries::hints(candidate, &checked.entries)?.try_iter()? { + let checked = crate::planner_migrations::_helpers::column::column_entries::checked( + &runtime, candidate, decided, &physical, + )?; + for hint in crate::planner_migrations::_helpers::column::column_entries::hints( + candidate, + &checked.entries, + )? + .try_iter()? + { hints.push(hint?); } entries.extend(checked.entries); warnings.extend(checked.warnings); } - let policy = super::planning_boundary::missing_origin_policy(&runtime)?; + let policy = + crate::planner_migrations::_helpers::planning::planning_boundary::missing_origin_policy( + &runtime, + )?; for entry in &mut entries { *entry = replace(entry, &[("missing_origin_policy", policy.clone())])?; - if let Some(warning) = super::column_warnings::entry_warning(entry)? { + if let Some(warning) = + crate::planner_migrations::_helpers::column::column_warnings::entry_warning(entry)? + { warnings.push(warning); } } - let overlaid = super::column_overlay::overlay(&snapshot, &candidates, &entries, &inputs)?; - let snapshot = super::column_overlay::cursors((phase, columns), overlaid, &entries)?; + let overlaid = crate::planner_migrations::_helpers::column::column_overlay::overlay( + &snapshot, + &candidates, + &entries, + &inputs, + )?; + let snapshot = crate::planner_migrations::_helpers::column::column_overlay::cursors( + (phase, columns), + overlaid, + &entries, + )?; Ok(ColumnMigrationPlanning { snapshot, entries, @@ -133,7 +156,3 @@ pub(crate) fn plan_column_migrations<'py>( warnings, }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(plan_column_migrations_of, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs index ae22e7faae..58375264e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs @@ -1,8 +1,8 @@ //! Preserve column-migration diagnostic codes and public messages. -use crate::bindings::_helpers::planner_migrations::column_objects::native; -use crate::bindings::_helpers::planner_migrations::column_objects::text; -use crate::bindings::_helpers::planner_migrations::constants::{ +use crate::planner_migrations::_helpers::column::column_objects::native; +use crate::planner_migrations::_helpers::column::column_objects::text; +use crate::planner_migrations::constants::{ DONE, MISSING_ORIGIN_ALLOW_OUTCOME, MISSING_ORIGIN_CONFIRMATION_OUTCOME, MISSING_ORIGIN_DENY_OUTCOME, }; @@ -14,7 +14,7 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -pub(super) fn warning<'py>( +pub(crate) fn warning<'py>( entry: &Bound<'py, PyAny>, code: &str, message: &str, @@ -40,7 +40,7 @@ pub(super) fn warning<'py>( ) } -pub(super) fn entry_warning<'py>(entry: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn entry_warning<'py>(entry: &Bound<'py, PyAny>) -> PyResult>> { let destination = native_field(entry, "destination")?; let qualified = native_field(&destination, "qualified_name")?; let relation: String = if qualified.is_truthy()? { @@ -94,7 +94,7 @@ pub(super) fn entry_warning<'py>(entry: &Bound<'py, PyAny>) -> PyResult( +pub(crate) fn unavailable<'py>( entry: &Bound<'py, PyAny>, reason: &str, ) -> PyResult> { @@ -117,7 +117,7 @@ pub(super) fn unavailable<'py>( } /// The build outcome a missing migration origin has under its policy. -pub(super) fn missing_origin_outcome(policy: &Bound<'_, PyAny>) -> PyResult<&'static str> { +pub(crate) fn missing_origin_outcome(policy: &Bound<'_, PyAny>) -> PyResult<&'static str> { match policy.extract::()?.as_str() { "allow" => Ok(MISSING_ORIGIN_ALLOW_OUTCOME), "require_confirmation" => Ok(MISSING_ORIGIN_CONFIRMATION_OUTCOME), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs index 7f1a140fdc..401798185f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs @@ -2,9 +2,9 @@ use std::collections::BTreeSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::planner_migrations::main::{ identical_renames as native_renames, rename_hints as native_hints, @@ -70,7 +70,7 @@ pub(crate) fn identical_renames<'py>( for (old, new) in pairs { renames.set_item(old, new)?; } - super::projections::explain( + crate::planner_migrations::_helpers::projections::query_shapes::explain( &previous, ¤t, &renames.into_any(), @@ -141,8 +141,3 @@ pub(crate) fn rename_hints<'py>( } PyTuple::new(py, result) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(identical_renames, module)?)?; - module.add_function(wrap_pyfunction!(rename_hints, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/known_input_columns.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/known_input_columns.rs diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/mod.rs new file mode 100644 index 0000000000..e244c0e49d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod column_candidates; +pub(crate) mod column_entries; +pub(crate) mod column_objects; +pub(crate) mod column_overlay; +pub(crate) mod column_planning; +pub(crate) mod column_warnings; +pub(crate) mod columns; +pub(crate) mod known_input_columns; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs index dc9590f417..ddfd3b7cfa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs @@ -2,8 +2,8 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::planner_migrations::column_objects::config; -use crate::bindings::_helpers::planner_migrations::models::ModelMigrationDeclaration; +use crate::planner_migrations::_helpers::column::column_objects::config; +use crate::planner_migrations::models::ModelMigrationDeclaration; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -12,17 +12,17 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_move; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; -pub(super) fn identity(item: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn identity(item: &Bound<'_, PyAny>) -> PyResult { let materialized = config(item, "materialized")?; Ok(materialized.eq("table")? || materialized.eq("view")?) } -pub(super) fn history(item: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn history(item: &Bound<'_, PyAny>) -> PyResult { let materialized = config(item, "materialized")?; Ok(materialized.eq("incremental")? || materialized.eq("snapshot")?) } -pub(super) fn newest<'py>( +pub(crate) fn newest<'py>( events: &Bound<'py, PyAny>, location: &Bound<'py, PyAny>, ) -> PyResult> { @@ -31,11 +31,11 @@ pub(super) fn newest<'py>( newest_move(events, &relation) } -pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) } -pub(super) fn declared<'py>( +pub(crate) fn declared<'py>( declarations: &[ModelMigrationDeclaration<'py>], state: &Bound<'py, PyAny>, ) -> PyResult> { @@ -63,7 +63,7 @@ pub(super) fn declared<'py>( Ok(result) } -pub(super) fn renames<'py>( +pub(crate) fn renames<'py>( runtime: &Bound<'py, PyAny>, names: &Bound<'py, PyDict>, state: &Bound<'py, PyAny>, @@ -101,7 +101,7 @@ pub(super) fn renames<'py>( Ok(result) } -pub(super) fn resumed<'py>( +pub(crate) fn resumed<'py>( runtime: &Bound<'py, PyAny>, models: &[Bound<'py, PyAny>], unbuilt: &Bound<'py, PyDict>, @@ -133,7 +133,7 @@ pub(super) fn resumed<'py>( Ok(result) } -pub(super) fn key<'py>(location: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn key<'py>(location: &Bound<'py, PyAny>) -> PyResult> { let schema = native_field(location, "schema")?; let schema = if schema.is_truthy()? { schema diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_models.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_models.rs index ad16642e95..7b84223fd1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_models.rs @@ -2,9 +2,9 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::planner_identity::metadata::build_function_local_hashes; -use crate::bindings::_helpers::planner_identity::metadata::build_model_version_identity_metadata_json; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_identity::main::build_function_local_hashes::build_function_local_hashes; +use crate::planner_identity::main::build_model_version_identity_metadata_json::build_model_version_identity_metadata_json; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; @@ -21,7 +21,7 @@ use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::BTreeMap; -pub(super) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult>> { let mut models: BTreeMap> = BTreeMap::new(); let mut dependencies: BTreeMap> = BTreeMap::new(); for item in compiled_models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { @@ -43,7 +43,7 @@ pub(super) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult], scope: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>, @@ -65,7 +65,7 @@ pub(super) fn triggered( Ok(false) } -pub(super) fn metadata<'py>( +pub(crate) fn metadata<'py>( runtime: &Bound<'py, PyAny>, models: &[Bound<'py, PyAny>], ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs index 9af31f02f2..48b937d2e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs @@ -1,13 +1,11 @@ //! Live candidate inspection and automatic request materialization. -use crate::bindings::_helpers::planner_migrations::column_objects::config; -use crate::bindings::_helpers::planner_migrations::column_objects::native; -use crate::bindings::_helpers::planner_migrations::discovery_events::{ +use crate::planner_migrations::_helpers::column::column_objects::config; +use crate::planner_migrations::_helpers::column::column_objects::native; +use crate::planner_migrations::_helpers::discovery::discovery_events::{ identity, key, newest, relation, }; -use crate::bindings::_helpers::planner_migrations::models::{ - ModelMigrationDeclaration, ModelMigrationRequest, -}; +use crate::planner_migrations::models::{ModelMigrationDeclaration, ModelMigrationRequest}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PySet, PySetMethods, PyString, PyTuple, @@ -49,10 +47,10 @@ fn location<'py>( ("name", name), ] }; - super::planning_origin::location(runtime, &fields) + crate::planner_migrations::_helpers::planning::planning_origin::location(runtime, &fields) } -pub(super) fn candidates<'py>( +pub(crate) fn candidates<'py>( runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, names: &Bound<'py, PyDict>, @@ -109,7 +107,7 @@ pub(super) fn candidates<'py>( PyTuple::new(py, result) } -pub(super) fn automatic<'py>( +pub(crate) fn automatic<'py>( runtime: &Bound<'py, PyAny>, scope: &Bound<'py, PyAny>, models: &[Bound<'py, PyAny>], @@ -159,7 +157,7 @@ pub(super) fn automatic<'py>( Ok(requests) } -pub(super) fn warnings<'py>( +pub(crate) fn warnings<'py>( models: &[Bound<'py, PyAny>], scope: &Bound<'py, PyAny>, ambiguous: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/mod.rs new file mode 100644 index 0000000000..b4f9da8864 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod discovery_events; +pub(crate) mod discovery_models; +pub(crate) mod discovery_requests; +pub(crate) mod model_discovery; +pub(crate) mod state_inspection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs similarity index 70% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs index c9164a2d4e..6f81614411 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs @@ -1,14 +1,14 @@ //! Native migration discovery over existing inspection and fingerprint cache boundaries. -use crate::bindings::_helpers::planner_migrations::matching::{ +use crate::planner_migrations::_helpers::discovery::discovery_events::renames; +use crate::planner_migrations::_helpers::discovery::discovery_models::ordered; +use crate::planner_migrations::_helpers::discovery::discovery_requests::automatic; +use crate::planner_migrations::_helpers::planning::matching::{ match_migration_fingerprints, migration_destination_fingerprints, }; -use crate::bindings::_helpers::planner_migrations::models::{ +use crate::planner_migrations::models::{ DiscoveryRequest, FingerprintInputs, ModelMigrationDiscovery, }; -use crate::bindings::_helpers::planner_migrations::{ - discovery_events, discovery_models, discovery_requests, -}; use pyo3::prelude::{PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple, PyTupleMethods}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -17,7 +17,7 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::name_object: use std::collections::BTreeSet; /// Manual and automatically discovered migration requests with their destination fingerprints. -pub(super) fn discover_model_migrations<'py>( +pub(crate) fn discover_model_migrations<'py>( request: DiscoveryRequest<'py>, ) -> PyResult> { let DiscoveryRequest { @@ -31,12 +31,17 @@ pub(super) fn discover_model_migrations<'py>( fingerprints: cache, } = request; let py = runtime.py(); - let models = discovery_models::ordered(&runtime)?; + let models = ordered(&runtime)?; let declared_names = PyDict::new(py); for declaration in &declarations { declared_names.set_item(&declaration.model_name, py.None())?; } - let triggered = discovery_models::triggered(&models, &scope, &snapshot, &declared_names)?; + let triggered = crate::planner_migrations::_helpers::discovery::discovery_models::triggered( + &models, + &scope, + &snapshot, + &declared_names, + )?; if !triggered && manual.is_empty() { return Ok(ModelMigrationDiscovery { requests: Vec::new(), @@ -44,7 +49,9 @@ pub(super) fn discover_model_migrations<'py>( destination_fingerprints: PyDict::new(py), }); } - let metadata = discovery_models::metadata(&runtime, &models)?; + let metadata = crate::planner_migrations::_helpers::discovery::discovery_models::metadata( + &runtime, &models, + )?; let inputs = FingerprintInputs { metadata_jsons: metadata.into_any(), dialect: native_field(&runtime, "adapter")?.call_method0("sql_analysis_dialect")?, @@ -54,12 +61,15 @@ pub(super) fn discover_model_migrations<'py>( &native_field(&state, "inspect_schemas")?, &[("schemas", project_schemas)], )?; - let declared = discovery_events::declared(&declarations, &state)?; + let declared = crate::planner_migrations::_helpers::discovery::discovery_events::declared( + &declarations, + &state, + )?; let names = PyDict::new(py); for item in &models { names.set_item(name_object(item)?, item)?; } - let event_renames = discovery_events::renames(&runtime, &names, &state)?; + let event_renames = renames(&runtime, &names, &state)?; for (name, origin) in declared.iter() { event_renames.set_item(name, origin)?; } @@ -86,9 +96,16 @@ pub(super) fn discover_model_migrations<'py>( unbuilt.set_item(name, item)?; } } - let resumed = discovery_events::resumed(&runtime, &models, &unbuilt, &state)?; + let resumed = crate::planner_migrations::_helpers::discovery::discovery_events::resumed( + &runtime, &models, &unbuilt, &state, + )?; let candidates = - discovery_requests::candidates(&runtime, &state, &names, (&declarations, &resumed))?; + crate::planner_migrations::_helpers::discovery::discovery_requests::candidates( + &runtime, + &state, + &names, + (&declarations, &resumed), + )?; for (name, _) in resumed.iter() { unbuilt.del_item(name)?; } @@ -100,13 +117,15 @@ pub(super) fn discover_model_migrations<'py>( let (variant_maps, matches, ambiguous) = match_migration_fingerprints((&models, new_names), &seeds, candidates.as_any(), &inputs)?; let mut requests = manual; - requests.extend(discovery_requests::automatic( + requests.extend(automatic( &runtime, &scope, &models, (matches.as_any(), &resumed), )?); - let warnings = discovery_requests::warnings(&models, &scope, &ambiguous)?; + let warnings = crate::planner_migrations::_helpers::discovery::discovery_requests::warnings( + &models, &scope, &ambiguous, + )?; let destination_fingerprints = migration_destination_fingerprints(&requests, &variant_maps, &inputs)?; Ok(ModelMigrationDiscovery { diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs new file mode 100644 index 0000000000..6429ff9e66 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs @@ -0,0 +1,111 @@ +//! Read-only warehouse evidence gathered once for all model migrations in one plan. + +use crate::planner_migrations::models::MigrationStateInspection; +use std::collections::HashSet; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyString, PyTuple}; +use sqlbuild_state::events::main::newest_fingerprint::newest_fingerprint; +use sqlbuild_state::events::models::{EventRelation, FingerprintKey, FingerprintOrder}; + +use crate::planner_phases::main::inputs::runtime_field::runtime_field; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A lowercased schema and relation name. +pub(crate) type RelationKey = (String, String); + +pub(crate) fn lowered(value: &Bound<'_, PyAny>) -> PyResult { + value.call_method0("lower")?.extract() +} + +pub(crate) fn schema_or_empty<'py>(value: Bound<'py, PyAny>) -> Bound<'py, PyAny> { + if value.is_none() { + PyString::new(value.py(), "").into_any() + } else { + value + } +} + +/// `((schema or "").lower(), name.lower())` of a relation or location. +pub(crate) fn relation_key(value: &Bound<'_, PyAny>) -> PyResult { + Ok(( + lowered(&schema_or_empty(native_field(value, "schema")?))?, + lowered(&native_field(value, "name")?)?, + )) +} + +/// The requested schemas not inspected yet, sorted, each with its lowercased key. +pub(crate) fn pending<'py>( + schemas: &Bound<'py, PyAny>, + inspected: &HashSet, +) -> PyResult, String)>> { + let mut pending: Vec<(String, Bound<'py, PyAny>, String)> = Vec::new(); + for schema in schemas.try_iter()? { + let schema: Bound<'py, PyAny> = schema?; + let key: String = lowered(&schema)?; + if !inspected.contains(&key) { + pending.push((schema.extract()?, schema, key)); + } + } + pending.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(pending + .into_iter() + .map(|(_, schema, key)| (schema, key)) + .collect()) +} + +pub(crate) fn tuple_of<'py>(py: Python<'py>, items: &[Py]) -> PyResult> { + PyTuple::new(py, items.iter().map(|item| item.bind(py))) +} + +fn fingerprint_key(fingerprint: &Bound<'_, PyAny>) -> PyResult { + let target_name: Bound<'_, PyAny> = native_field(fingerprint, "target_name")?; + let target: Option = if target_name.is_none() { + None + } else { + Some(EventRelation { + database: native_field(fingerprint, "target_database")?.extract()?, + schema: native_field(fingerprint, "target_schema")?.extract()?, + name: target_name.extract()?, + }) + }; + Ok(FingerprintKey { + order: FingerprintOrder { + timestamp: native_field(fingerprint, "ts")? + .call_method0("isoformat")? + .extract()?, + run_id: native_field(fingerprint, "run_id")?.extract()?, + }, + node_name: native_field(fingerprint, "node_name")?.extract()?, + target, + }) +} + +pub(crate) fn newest<'py>( + py: Python<'py>, + candidates: &[Py], + (target, model_name): (Option<&EventRelation>, Option<&str>), +) -> PyResult>> { + let keys: Vec = candidates + .iter() + .map(|candidate| fingerprint_key(candidate.bind(py))) + .collect::>()?; + Ok(newest_fingerprint(&keys, target, model_name) + .map(|index| candidates[index].bind(py).clone())) +} + +/// A fresh live-state inspection over the runtime's connection and planning database. +pub(crate) fn inspection<'py>( + runtime: &Bound<'py, PyAny>, + known_fingerprints: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let state = MigrationStateInspection::new( + &runtime_field(runtime, |runtime| &runtime.adapter)?, + &runtime_field(runtime, |runtime| &runtime.connection)?, + &crate::planner_migrations::_helpers::planning::planning_origin::planning_database( + runtime, + )?, + known_fingerprints, + )?; + Ok(Bound::new(runtime.py(), state)?.into_any()) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/mod.rs new file mode 100644 index 0000000000..6815150f76 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod column; +pub(crate) mod discovery; +pub(crate) mod model; +pub(crate) mod old_name; +pub(crate) mod planning; +pub(crate) mod projections; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/mod.rs new file mode 100644 index 0000000000..b6317c802b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod model_entries; +pub(crate) mod model_handover; +pub(crate) mod model_overlay; +pub(crate) mod model_planning; +pub(crate) mod model_warnings; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs index 9e7f22bb28..79828786c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs @@ -2,14 +2,12 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::planner_migrations::column_objects::{lower, model, replace, text}; -use crate::bindings::_helpers::planner_migrations::compatibility::check_migration_compatibility; -use crate::bindings::_helpers::planner_migrations::model_handover::{ +use crate::planner_migrations::_helpers::column::column_objects::{lower, model, replace, text}; +use crate::planner_migrations::_helpers::model::model_handover::{ applies, empty_version, handover, }; -use crate::bindings::_helpers::planner_migrations::models::{ - CompatibilityCheck, MigrationDecisionRequest, -}; +use crate::planner_migrations::_helpers::planning::compatibility::check_migration_compatibility; +use crate::planner_migrations::models::{CompatibilityCheck, MigrationDecisionRequest}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyBool; use sqlbuild_analysis::planner_migrations::main::model_decision::model_decision; @@ -24,7 +22,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_move; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; -pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) } @@ -40,7 +38,7 @@ fn key(location: &Bound<'_, PyAny>) -> PyResult<(String, String)> { )) } -pub(super) fn decision<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult> { +pub(crate) fn decision<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult> { replace( entry, &[( @@ -63,13 +61,13 @@ fn result<'py>( } /// The decided plan entry, the inherited fingerprint, and whether the history handover applies. -pub(super) struct MigrationDecision<'py> { +pub(crate) struct MigrationDecision<'py> { pub(super) entry: Bound<'py, PyAny>, pub(super) inherited: Bound<'py, PyAny>, pub(super) applies: bool, } -pub(super) fn decide<'py>( +pub(crate) fn decide<'py>( decision_request: &MigrationDecisionRequest<'_, 'py>, ) -> PyResult> { let runtime = decision_request.runtime.clone(); @@ -263,7 +261,7 @@ pub(super) fn decide<'py>( &native_field(&entry, "origin_version_hash")?, )?; result( - super::planning_execution::migration_with_execution(&runtime, &item, &entry)?, + crate::planner_migrations::_helpers::planning::planning_execution::migration_with_execution(&runtime, &item, &entry)?, inherited, true, ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs index cd6e60dc9c..e63588b412 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs @@ -1,7 +1,7 @@ //! Adapt immutable migration fingerprints without changing their persisted representation. -use crate::bindings::_helpers::planner_migrations::column_objects::{replace, text}; -use crate::bindings::_helpers::planner_migrations::constants::MIGRATION_MODEL_NAME_METADATA_KEY; +use crate::planner_migrations::_helpers::column::column_objects::{replace, text}; +use crate::planner_migrations::constants::MIGRATION_MODEL_NAME_METADATA_KEY; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::boundary::main::imported::imported; @@ -12,7 +12,7 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::q use sqlbuild_python_project::fingerprints::main::compute_query_hash::compute_query_hash; use sqlbuild_python_project::planner_phases::main::object::object; -pub(super) fn applies( +pub(crate) fn applies( model: &Bound<'_, PyAny>, event: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>, @@ -24,7 +24,7 @@ pub(super) fn applies( < presumed_utc(&native_field(event, "created_at")?)?) } -pub(super) fn handover<'py>( +pub(crate) fn handover<'py>( model: &Bound<'py, PyAny>, fingerprint: &Bound<'py, PyAny>, version: &Bound<'py, PyAny>, @@ -77,7 +77,7 @@ pub(super) fn handover<'py>( ) } -pub(super) fn equivalent<'py>( +pub(crate) fn equivalent<'py>( model: &Bound<'py, PyAny>, handover: Bound<'py, PyAny>, destinations: &Bound<'py, PyAny>, @@ -85,7 +85,11 @@ pub(super) fn equivalent<'py>( ) -> PyResult> { let py = model.py(); if handover.is_none() - || !destinations.contains(super::matching::stored_migration_fingerprint(&handover)?)? + || !destinations.contains( + crate::planner_migrations::_helpers::planning::matching::stored_migration_fingerprint( + &handover, + )?, + )? { return Ok(handover); } @@ -108,7 +112,7 @@ pub(super) fn equivalent<'py>( ) } -pub(super) fn empty_version<'py>( +pub(crate) fn empty_version<'py>( model: &Bound<'py, PyAny>, fingerprint: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs index 41931dd473..709519a2ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs @@ -1,15 +1,15 @@ //! Project inherited fingerprints and live relation facts before normal planning. -use crate::bindings::_helpers::planner_migrations::column_objects::native; -use crate::bindings::_helpers::planner_migrations::column_objects::{config, replace}; -use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_migrations::_helpers::column::column_objects::native; +use crate::planner_migrations::_helpers::column::column_objects::{config, replace}; +use crate::planner_migrations::models::MigrationPhase; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -pub(super) fn overlay<'py>( +pub(crate) fn overlay<'py>( phase: &MigrationPhase<'py>, state: &Bound<'py, PyAny>, entries: &[Bound<'py, PyAny>], @@ -80,7 +80,7 @@ pub(super) fn overlay<'py>( let cursors = native_field(&snapshot, "cursor_snapshots")? .cast_into::()? .copy()?; - let gathered = super::planning_boundary::redirected_cursor_snapshots( + let gathered = crate::planner_migrations::_helpers::planning::planning_boundary::redirected_cursor_snapshots( phase, (relations.clone().into_any(), redirected.into_any()), None, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs similarity index 83% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs index 4f9a734749..4cc0a60a9a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs @@ -1,16 +1,16 @@ //! Model migration orchestration over Python-owned live inspection. -use crate::bindings::_helpers::planner_changes::renames::plan_reference_renames; -use crate::bindings::_helpers::planner_migrations::column_objects::{config, replace}; -use crate::bindings::_helpers::planner_migrations::discovery::discover_model_migrations; -use crate::bindings::_helpers::planner_migrations::models::{ +use crate::planner_changes::main::plan_reference_renames::plan_reference_renames; +use crate::planner_migrations::_helpers::column::column_objects::{config, replace}; +use crate::planner_migrations::_helpers::discovery::model_discovery::discover_model_migrations; +use crate::planner_migrations::_helpers::planning::planning_origin::{ + require_same_database, resolve_origin, +}; +use crate::planner_migrations::models::{ DiscoveryRequest, MigrationDecisionRequest, MigrationPhase, ModelMigrationDeclaration, ModelMigrationDiscovery, ModelMigrationPlanning, ModelMigrationRequest, }; -use crate::bindings::_helpers::planner_migrations::planning_origin::{ - require_same_database, resolve_origin, -}; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyString, PyTuple}; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; @@ -77,23 +77,7 @@ fn manual_migration_requests<'py>( Ok(requests) } -/// Model migrations of the selected scope: discovered and declared moves, decided against live state. -pub(crate) fn plan_model_migrations<'py>( - phase: &MigrationPhase<'py>, - fingerprints: &Bound<'py, PyAny>, -) -> PyResult> { - let runtime = phase.runtime.clone(); - let snapshot = phase.snapshot.clone(); - let schemas = super::planning_origin::project_schemas(&runtime)?; - let state = super::state_inspection::inspection( - &runtime, - Some(&native_field(&snapshot, "fingerprints")?), - )?; - let discovery = discover((phase, fingerprints), &state, &schemas)?; - finish(phase, &state, &schemas, &discovery) -} - -fn discover<'py>( +pub(crate) fn discover<'py>( (phase, fingerprints): (&MigrationPhase<'py>, &Bound<'py, PyAny>), state: &Bound<'py, PyAny>, schemas: &Bound<'py, PyAny>, @@ -136,7 +120,7 @@ fn discover<'py>( }) } -fn finish<'py>( +pub(crate) fn finish<'py>( phase: &MigrationPhase<'py>, state: &Bound<'py, PyAny>, schemas: &Bound<'py, PyAny>, @@ -182,19 +166,21 @@ fn finish<'py>( } else { native_field(&project_config, "snapshots")? }; - let policy = super::planning_boundary::missing_origin_policy(&runtime)?; + let policy = crate::planner_migrations::_helpers::planning::planning_boundary::missing_origin_policy(&runtime)?; for (request, origin, origin_model) in resolved { let item = request.model.clone(); let name = name_object(&item)?; - let decided = super::model_entries::decide(&MigrationDecisionRequest { - runtime: runtime.clone(), - request, - origin, - origin_model, - snapshot: snapshot.clone(), - state: state.clone(), - snapshots_config: snapshots_config.clone(), - })?; + let decided = crate::planner_migrations::_helpers::model::model_entries::decide( + &MigrationDecisionRequest { + runtime: runtime.clone(), + request, + origin, + origin_model, + snapshot: snapshot.clone(), + state: state.clone(), + snapshots_config: snapshots_config.clone(), + }, + )?; entries.push(replace( &decided.entry, &[("missing_origin_policy", policy.clone())], @@ -207,7 +193,7 @@ fn finish<'py>( )?; handovers.set_item( name, - super::model_handover::equivalent( + crate::planner_migrations::_helpers::model::model_handover::equivalent( &item, decided.inherited, &destinations, @@ -232,14 +218,18 @@ fn finish<'py>( planned.set_item(&name, origin_model)?; } } - if let Some(warning) = super::model_warnings::entry_warning(entry)? { + if let Some(warning) = + crate::planner_migrations::_helpers::model::model_warnings::entry_warning(entry)? + { warnings.push(warning); } } let overlaid = if entries.is_empty() { snapshot.clone() } else { - super::model_overlay::overlay(phase, state, &entries, &handovers)? + crate::planner_migrations::_helpers::model::model_overlay::overlay( + phase, state, &entries, &handovers, + )? }; let reference_renames = object( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs index e2836b5359..c29e3a9b4f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs @@ -1,16 +1,16 @@ //! Adapt diagnostic display facts while preserving policy and code ordering. -use crate::bindings::_helpers::planner_migrations::column_warnings::{ +use crate::planner_migrations::_helpers::column::column_warnings::{ missing_origin_outcome, warning, }; -use crate::bindings::_helpers::planner_migrations::constants::HIDDEN_ORIGIN_REMEDY; +use crate::planner_migrations::constants::HIDDEN_ORIGIN_REMEDY; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyString; use sqlbuild_analysis::planner_migrations::main::model_warning::model_warning; use sqlbuild_analysis::planner_migrations::models::ModelWarning; use sqlbuild_python_base::records::main::fields::field::field as native_field; -pub(super) fn entry_warning<'py>(entry: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn entry_warning<'py>(entry: &Bound<'py, PyAny>) -> PyResult>> { let py = entry.py(); let name: String = native_field(entry, "model_name")?.extract()?; let origin = display(&native_field(entry, "origin")?)?; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/mod.rs new file mode 100644 index 0000000000..87e886af43 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod old_name_entries; +pub(crate) mod old_name_objects; +pub(crate) mod old_name_planning; +pub(crate) mod old_name_references; +pub(crate) mod old_names; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs index eb374c54d2..18f8f935b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs @@ -1,14 +1,14 @@ //! Assemble new and resumed old-name view steps from recorded facts. -use crate::bindings::_helpers::planner_migrations::column_objects::{model, text}; -use crate::bindings::_helpers::planner_migrations::constants::{PENDING_ARCHIVE, PENDING_VIEW}; -use crate::bindings::_helpers::planner_migrations::models::OldNamePlanner; -use crate::bindings::_helpers::planner_migrations::old_name_objects::{ +use crate::planner_migrations::_helpers::column::column_objects::{model, text}; +use crate::planner_migrations::_helpers::old_name::old_name_objects::{ expires, location, relation, retention, stores_history, }; -use crate::bindings::_helpers::planner_migrations::old_names::declared_column_aliases; -use crate::bindings::_helpers::planner_migrations::old_names::old_name_view_skip_reason; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_migrations::_helpers::old_name::old_names::declared_column_aliases; +use crate::planner_migrations::_helpers::old_name::old_names::old_name_view_skip_reason; +use crate::planner_migrations::constants::{PENDING_ARCHIVE, PENDING_VIEW}; +use crate::planner_migrations::models::OldNamePlanner; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyBool, PyList, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs index 4f7811e34b..7518d1ea03 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs @@ -2,7 +2,7 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::planner_migrations::column_objects::{config, model}; +use crate::planner_migrations::_helpers::column::column_objects::{config, model}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -13,11 +13,11 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::state_rows::main::migrations::old_name_retention::old_name_retention; use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; -pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) } -pub(super) fn location<'py>( +pub(crate) fn location<'py>( runtime: &Bound<'py, PyAny>, relation: &Bound<'py, PyAny>, ) -> PyResult> { @@ -34,12 +34,12 @@ pub(super) fn location<'py>( new_record_with(runtime.py(), RecordId::CompiledRelationLocation, &fields) } -pub(super) fn stores_history(item: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn stores_history(item: &Bound<'_, PyAny>) -> PyResult { let materialization = config(item, "materialized")?; Ok(materialization.eq("incremental")? || materialization.eq("snapshot")?) } -pub(super) fn retention<'py>( +pub(crate) fn retention<'py>( item: &Bound<'py, PyAny>, project: &Bound<'py, PyAny>, ) -> PyResult> { @@ -52,7 +52,7 @@ pub(super) fn retention<'py>( ) } -pub(super) fn expires<'py>( +pub(crate) fn expires<'py>( retention: &Bound<'py, PyAny>, now: &Bound<'py, PyAny>, ) -> PyResult> { @@ -65,7 +65,7 @@ pub(super) fn expires<'py>( } } -pub(super) fn recorded_view<'py>( +pub(crate) fn recorded_view<'py>( runtime: &Bound<'py, PyAny>, history: &Bound<'py, PyAny>, models: &[Bound<'py, PyAny>], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs index 4dbe2e9c79..5f591d5791 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs @@ -2,11 +2,11 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; -use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; -use crate::bindings::_helpers::planner_migrations::old_names::old_name_claim_errors; -use crate::bindings::_helpers::planner_migrations::old_names::old_name_schemas_to_read; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_migrations::_helpers::old_name::old_name_objects::recorded_view; +use crate::planner_migrations::_helpers::old_name::old_names::old_name_claim_errors; +use crate::planner_migrations::_helpers::old_name::old_names::old_name_schemas_to_read; +use crate::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PySet, PySetMethods, PyTuple}; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; @@ -62,7 +62,10 @@ fn schemas<'py>( } } if include_project { - let project = super::planning_origin::project_schemas(&runtime)?; + let project = + crate::planner_migrations::_helpers::planning::planning_origin::project_schemas( + &runtime, + )?; for schema in project.try_iter()? { needed.add(schema?)?; } @@ -98,7 +101,9 @@ pub(crate) fn plan_old_name_views<'py>( selected.push(item.clone()); } } - let state = super::state_inspection::inspection(&runtime, None)?; + let state = crate::planner_migrations::_helpers::discovery::state_inspection::inspection( + &runtime, None, + )?; let needed = schemas((phase, moves.0), &selected)?; if needed.is_truthy()? { call_with( @@ -123,7 +128,7 @@ pub(crate) fn plan_old_name_views<'py>( histories.push(history); } let views = PyTuple::new(py, views)?.into_any(); - let planner = super::models::OldNamePlanner::new( + let planner = crate::planner_migrations::models::OldNamePlanner::new( (&phase.runtime, moves.1), models, histories, @@ -168,9 +173,11 @@ pub(crate) fn plan_old_name_views<'py>( )?, )?; let mut warnings: Vec> = claims.try_iter()?.collect::>()?; - warnings.extend(super::old_name_references::errors( - &runtime, &views, &entries, - )?); + warnings.extend( + crate::planner_migrations::_helpers::old_name::old_name_references::errors( + &runtime, &views, &entries, + )?, + ); Ok(OldNameViewPlanning { entries, views, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs index 4a6a085759..4607ee3b2d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs @@ -2,7 +2,7 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::planner_migrations::column_objects::{native, text}; +use crate::planner_migrations::_helpers::column::column_objects::{native, text}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -49,7 +49,7 @@ fn old_relation<'py>( ) } -pub(super) fn errors<'py>( +pub(crate) fn errors<'py>( runtime: &Bound<'py, PyAny>, views: &Bound<'py, PyAny>, entries: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs index 1f5087138f..b870bbb145 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs @@ -2,7 +2,7 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::planner_migrations::old_name_objects::relation; +use crate::planner_migrations::_helpers::old_name::old_name_objects::relation; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PySet, PyTuple}; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/compatibility.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/compatibility.rs index fd9d453326..37c9a370ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/compatibility.rs @@ -1,9 +1,7 @@ //! Convert existing planner contracts for origin compatibility checks. -use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; -use crate::bindings::_helpers::planner_migrations::models::{ - CompatibilityCheck, MigrationCompatibilityResult, -}; +use crate::planner_changes::main::detect_model_changes::detect_model_changes; +use crate::planner_migrations::models::{CompatibilityCheck, MigrationCompatibilityResult}; use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -74,7 +72,7 @@ fn blocking<'py>( ) } /// Whether an origin relation's columns can carry the destination model's history. -pub(super) fn check_migration_compatibility<'py>( +pub(crate) fn check_migration_compatibility<'py>( check: &CompatibilityCheck<'py>, ) -> PyResult> { let py = check.model.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs index 821272dfc1..26d7df98bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs @@ -1,8 +1,6 @@ //! Fingerprint cache callbacks around native one-to-one migration matching. -use crate::bindings::_helpers::planner_migrations::models::{ - FingerprintInputs, ModelMigrationRequest, -}; +use crate::planner_migrations::models::{FingerprintInputs, ModelMigrationRequest}; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -46,7 +44,7 @@ fn variants<'py>( } /// The fingerprint variants tried, the matched candidate per new model, and ambiguous names. -pub(super) fn match_migration_fingerprints<'py>( +pub(crate) fn match_migration_fingerprints<'py>( (models, new_names): (&[Bound<'py, PyAny>], BTreeSet), seed_variants: &Bound<'py, PyAny>, candidates: &Bound<'py, PyAny>, @@ -86,7 +84,7 @@ pub(super) fn match_migration_fingerprints<'py>( } /// The fingerprint variants of each requested destination model with metadata. -pub(super) fn migration_destination_fingerprints<'py>( +pub(crate) fn migration_destination_fingerprints<'py>( requests: &[ModelMigrationRequest<'py>], variant_maps: &Bound<'py, PyAny>, inputs: &FingerprintInputs<'py>, @@ -108,7 +106,7 @@ pub(super) fn migration_destination_fingerprints<'py>( } /// The migration fingerprint stored in a model fingerprint's metadata. -pub(super) fn stored_migration_fingerprint( +pub(crate) fn stored_migration_fingerprint( fingerprint: &Bound<'_, PyAny>, ) -> PyResult> { let metadata: String = native_field(fingerprint, "metadata_json")?.extract()?; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/mod.rs new file mode 100644 index 0000000000..e8f2dc0ad7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod compatibility; +pub(crate) mod matching; +pub(crate) mod planning_boundary; +pub(crate) mod planning_execution; +pub(crate) mod planning_origin; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs similarity index 83% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs index 2257feefc1..9436b99e4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs @@ -1,18 +1,18 @@ //! Assemble migration policy and redirected live cursor inspection requests. -use crate::bindings::_helpers::planner_migrations::column_objects::native; -use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; -use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; -use crate::bindings::_helpers::planner_phases::retention::effective_target_config; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_warehouse::snapshot::gather_redirected_cursor_snapshots; +use crate::planner_migrations::_helpers::column::column_objects::native; +use crate::planner_migrations::models::MigrationPhase; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; +use crate::planner_phases::main::rebuilds::effective_target_config::effective_target_config; +use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; +use crate::planner_warehouse::main::inspection::gather_redirected_cursor_snapshots::gather_redirected_cursor_snapshots; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; /// The target's policy for migration origins that no longer exist. -pub(super) fn missing_origin_policy<'py>( +pub(crate) fn missing_origin_policy<'py>( runtime: &Bound<'py, PyAny>, ) -> PyResult> { native_field( @@ -22,7 +22,7 @@ pub(super) fn missing_origin_policy<'py>( } /// Live cursor snapshots of models whose history now sits in a redirected relation. -pub(super) fn redirected_cursor_snapshots<'py>( +pub(crate) fn redirected_cursor_snapshots<'py>( phase: &MigrationPhase<'py>, (existing_relations, target_relations): (Bound<'py, PyAny>, Bound<'py, PyAny>), origin_cursor_columns: Option>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs index e50f135a0e..bf918e5010 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs @@ -2,8 +2,8 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::planner_migrations::column_objects::replace; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_migrations::_helpers::column::column_objects::replace; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyBool; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -40,7 +40,7 @@ fn migration_promotion<'py>( } /// The migration entry with the adapter's staging, transfer and promotion choices. -pub(super) fn migration_with_execution<'py>( +pub(crate) fn migration_with_execution<'py>( runtime: &Bound<'py, PyAny>, model: &Bound<'py, PyAny>, entry: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs index 53fb78acba..c303de0661 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs @@ -1,8 +1,8 @@ //! Resolve authored migration origins at the adapter and inspection boundaries. -use crate::bindings::_helpers::planner_migrations::constants::QUALIFIED_RELATION_MAX_PARTS; -use crate::bindings::_helpers::planner_migrations::models::ModelMigrationRequest; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::planner_migrations::constants::QUALIFIED_RELATION_MAX_PARTS; +use crate::planner_migrations::models::ModelMigrationRequest; +use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::PyErr; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PySet, PySetMethods}; @@ -15,7 +15,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; -pub(super) fn location<'py>( +pub(crate) fn location<'py>( runtime: &Bound<'py, PyAny>, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { @@ -68,7 +68,7 @@ fn input_error(py: Python<'_>, message: String) -> PyResult { } /// The database migration planning inspects: the first model destination's, else the target's. -pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { let project = runtime_field(runtime, |runtime| &runtime.project)?; let fallback = native_field(&project, "effective_target_database")?; for item in compiled_models(&project)?.try_iter()? { @@ -81,7 +81,7 @@ pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult(runtime: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn project_schemas<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { let schemas = PySet::empty(runtime.py())?; for item in compiled_models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let schema = native_field(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&item?)?, "schema")?; @@ -93,7 +93,7 @@ pub(super) fn project_schemas<'py>(runtime: &Bound<'py, PyAny>) -> PyResult, location: &Bound<'_, PyAny>, ) -> PyResult<()> { @@ -124,7 +124,7 @@ pub(super) fn require_same_database( } /// The origin location and origin model name of one migration request. -pub(super) fn resolve_origin<'py>( +pub(crate) fn resolve_origin<'py>( request: &ModelMigrationRequest<'py>, runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, @@ -135,7 +135,7 @@ pub(super) fn resolve_origin<'py>( } let raw = match &request.raw_origin { Some(raw) if raw.is_truthy()? => raw.clone(), - _ => super::column_objects::text(py, ""), + _ => crate::planner_migrations::_helpers::column::column_objects::text(py, ""), }; let item = request.model.clone(); let destination = diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs new file mode 100644 index 0000000000..7b08211617 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs @@ -0,0 +1,42 @@ +//! Migration identity binding with unchanged metadata and hash format. + +use crate::planner_migrations::main::fingerprint::fingerprint; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use std::collections::BTreeMap; + +/// Hash normalized SQL and non-storage metadata through rename identities. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_migration_fingerprint( + py: Python<'_>, + options: Option>, +) -> PyResult> { + let arguments = KeywordArguments::read( + py, + "build_migration_fingerprint", + options, + ( + &["query_sql", "metadata_json", "ref_identities", "dialect"], + &[], + ), + )?; + let mut identities: BTreeMap = BTreeMap::new(); + for item in arguments + .value("ref_identities")? + .call_method0("items")? + .try_iter()? + { + let (key, value): (String, String) = item?.extract()?; + identities.insert(key, value); + } + let dialect: Option = arguments.value("dialect")?.extract()?; + fingerprint( + &arguments.value("query_sql")?.extract::()?, + &arguments.value("metadata_json")?.extract::()?, + &identities, + dialect.as_deref(), + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/getters.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/getters.rs similarity index 66% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/getters.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/getters.rs index 92bf934e48..6901d0c07c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/getters.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/getters.rs @@ -3,17 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::planner_migrations::state_inspection::MigrationStateInspection; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::planner_migrations::models::MigrationStateInspection; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { - class: class_of::, - read: read_migration_state_inspection, -}]; - -fn read_migration_state_inspection<'py>( +pub(crate) fn read_migration_state_inspection<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/mod.rs new file mode 100644 index 0000000000..76756461fb --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod fingerprint; +pub(crate) mod getters; +pub(crate) mod query_shapes; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/query_shapes.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/query_shapes.rs index d5a131042e..92db2b5325 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/query_shapes.rs @@ -1,10 +1,10 @@ //! Adapt existing Python query shape models to pure migration proofs. -use crate::bindings::_helpers::planner_migrations::errors::ProofError; +use crate::planner_migrations::errors::ProofError; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::planner_migrations::main::{query_shape, renames_explain_change as proof}; use sqlbuild_analysis::planner_migrations::models::{QueryProjection, QueryShape}; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -146,8 +146,3 @@ pub(crate) fn renames_explain_change( &arguments.value("input_columns")?, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_query_shape, module)?)?; - module.add_function(wrap_pyfunction!(renames_explain_change, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/constants.rs b/crates/sqlbuild-python-planner/src/planner_migrations/constants.rs new file mode 100644 index 0000000000..06519f2cb1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/constants.rs @@ -0,0 +1,38 @@ +//! Public constants of planner migrations. + +use crate::planner_migrations::_helpers::projections::getters::read_migration_state_inspection; +use crate::planner_migrations::models::MigrationStateInspection; +use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +pub(crate) const AUTOMATIC: &str = "automatic"; + +pub(crate) const DONE: &str = "done"; + +pub(crate) const PENDING_ARCHIVE: &str = "pending_archive"; + +pub(crate) const PENDING_VIEW: &str = "pending_view"; + +pub(crate) const LIST_RELATIONS_FOR_INSPECTION: AdapterHost = + AdapterHost::ListRelationsForInspection; + +pub(crate) const GET_COLUMNS_FOR_INSPECTION: AdapterHost = AdapterHost::GetColumnsForInspection; + +pub(crate) const QUALIFIED_RELATION_MAX_PARTS: usize = 3; + +pub(crate) const MIGRATION_MODEL_NAME_METADATA_KEY: &str = "model_name"; + +pub(crate) const HIDDEN_ORIGIN_REMEDY: &str = "the connection may lack permission to see it, or it was dropped outside SQLBuild; restore access to it, or remove migrate_from to build the model fresh deliberately (missing_migration_origin does not apply)"; + +pub(crate) const MISSING_ORIGIN_ALLOW_OUTCOME: &str = "nothing is migrated and the build continues as if migrate_from were absent; remove migrate_from once no target still needs it"; + +pub(crate) const MISSING_ORIGIN_CONFIRMATION_OUTCOME: &str = "building without it needs confirmation (missing_migration_origin = require_confirmation); pass --allow-missing-migration-origin, or remove migrate_from if it is no longer needed"; + +pub(crate) const MISSING_ORIGIN_DENY_OUTCOME: &str = "the build stops (missing_migration_origin = deny); if the migration already happened elsewhere or is no longer needed, remove migrate_from"; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[GetterEntry { + class: class_of::, + read: read_migration_state_inspection, +}]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/errors.rs b/crates/sqlbuild-python-planner/src/planner_migrations/errors.rs similarity index 69% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/errors.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/errors.rs index 2f26486686..53c8c47d73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/errors.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/errors.rs @@ -1,4 +1,4 @@ -//! Preserve input lookup exceptions across the pure proof callback. +//! Errors of migration planning: input lookup exceptions preserved across the pure proof callback. use pyo3::{PyErr, exceptions::PyValueError}; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/fingerprint.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/fingerprint.rs new file mode 100644 index 0000000000..24dcffc7e1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/fingerprint.rs @@ -0,0 +1,24 @@ +//! The migration fingerprint of SQL and metadata through rename identities, `generic` by default. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::planner_migrations::main::migration_fingerprint; +use std::collections::BTreeMap; + +/// The migration fingerprint of SQL and metadata through rename identities, `generic` by default. +pub fn fingerprint( + sql: &str, + metadata: &str, + identities: &BTreeMap, + dialect: Option<&str>, +) -> PyResult> { + migration_fingerprint::migration_fingerprint( + sql, + metadata, + identities, + dialect + .filter(|value| !value.is_empty()) + .unwrap_or("generic"), + ) + .map_err(PyValueError::new_err) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/migration_fingerprint_ref_names.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/migration_fingerprint_ref_names.rs new file mode 100644 index 0000000000..258fc977fc --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/migration_fingerprint_ref_names.rs @@ -0,0 +1,19 @@ +//! Every name whose rename identity can change the migration fingerprint. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::PyFrozenSet; +use sqlbuild_analysis::planner_migrations::main::migration_ref_names; + +/// Every name whose rename identity can change the migration fingerprint. +pub fn migration_fingerprint_ref_names<'py>( + py: Python<'py>, + query_sql: &str, + metadata_json: &str, +) -> PyResult> { + PyFrozenSet::new( + py, + migration_ref_names::migration_ref_names(query_sql, metadata_json) + .map_err(PyValueError::new_err)?, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/mod.rs new file mode 100644 index 0000000000..972c3e0206 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/mod.rs @@ -0,0 +1,10 @@ +pub mod fingerprint; +pub mod migration_fingerprint_ref_names; +pub mod plan_column_migrations; +pub mod plan_model_migrations; +pub mod plan_old_name_views; +pub mod register_column_planning; +pub mod register_columns; +pub mod register_fingerprint; +pub mod register_projections; +pub mod with_migration_fingerprints; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_column_migrations.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_column_migrations.rs new file mode 100644 index 0000000000..201f729356 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_column_migrations.rs @@ -0,0 +1,16 @@ +//! Decide column renames for the selected incremental and snapshot models. + +use crate::planner_migrations::_helpers::column::column_planning; +use crate::planner_migrations::models::{ + ColumnMigrationInputs, ColumnMigrationPlanning, MigrationPhase, +}; +use pyo3::prelude::PyResult; + +/// Decide column renames for the selected incremental and snapshot models. +#[inline] +pub fn plan_column_migrations<'py>( + phase: &MigrationPhase<'py>, + columns: &ColumnMigrationInputs<'py>, +) -> PyResult> { + column_planning::plan_column_migrations(phase, columns) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_model_migrations.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_model_migrations.rs new file mode 100644 index 0000000000..2409dc6fe7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_model_migrations.rs @@ -0,0 +1,24 @@ +//! Model migrations of the selected scope: discovered and declared moves, decided against live state. + +use crate::planner_migrations::_helpers::model::model_planning::discover; +use crate::planner_migrations::_helpers::model::model_planning::finish; +use crate::planner_migrations::models::{MigrationPhase, ModelMigrationPlanning}; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Model migrations of the selected scope: discovered and declared moves, decided against live state. +pub fn plan_model_migrations<'py>( + phase: &MigrationPhase<'py>, + fingerprints: &Bound<'py, PyAny>, +) -> PyResult> { + let runtime = phase.runtime.clone(); + let snapshot = phase.snapshot.clone(); + let schemas = + crate::planner_migrations::_helpers::planning::planning_origin::project_schemas(&runtime)?; + let state = crate::planner_migrations::_helpers::discovery::state_inspection::inspection( + &runtime, + Some(&native_field(&snapshot, "fingerprints")?), + )?; + let discovery = discover((phase, fingerprints), &state, &schemas)?; + finish(phase, &state, &schemas, &discovery) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_old_name_views.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_old_name_views.rs new file mode 100644 index 0000000000..4388949a57 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/plan_old_name_views.rs @@ -0,0 +1,14 @@ +//! Old-name compatibility views to create, resume or retire for the planned model and column migrations. + +use crate::planner_migrations::_helpers::old_name::old_name_planning; +use crate::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Old-name compatibility views to create, resume or retire for the planned model and column migrations. +#[inline] +pub fn plan_old_name_views<'py>( + phase: &MigrationPhase<'py>, + moves: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + old_name_planning::plan_old_name_views(phase, moves) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_column_planning.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_column_planning.rs new file mode 100644 index 0000000000..6cc5b02f6c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_column_planning.rs @@ -0,0 +1,10 @@ +//! Register the column planning functions and classes. + +use crate::planner_migrations::_helpers::column::column_planning::plan_column_migrations_of; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the column planning functions and classes. +pub fn register_column_planning(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(plan_column_migrations_of, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_columns.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_columns.rs new file mode 100644 index 0000000000..6d88a83b9b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_columns.rs @@ -0,0 +1,12 @@ +//! Register the columns functions and classes. + +use crate::planner_migrations::_helpers::column::columns::identical_renames; +use crate::planner_migrations::_helpers::column::columns::rename_hints; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the columns functions and classes. +pub fn register_columns(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(identical_renames, module)?)?; + module.add_function(wrap_pyfunction!(rename_hints, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_fingerprint.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_fingerprint.rs new file mode 100644 index 0000000000..f2477f5ceb --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_fingerprint.rs @@ -0,0 +1,11 @@ +//! Register the fingerprint functions and classes. + +use crate::planner_migrations::_helpers::projections::fingerprint::build_migration_fingerprint; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the fingerprint functions and classes. +pub fn register_fingerprint(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_migration_fingerprint, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/register_projections.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_projections.rs new file mode 100644 index 0000000000..275627e07b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/register_projections.rs @@ -0,0 +1,12 @@ +//! Register the projections functions and classes. + +use crate::planner_migrations::_helpers::projections::query_shapes::parse_query_shape; +use crate::planner_migrations::_helpers::projections::query_shapes::renames_explain_change; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the projections functions and classes. +pub fn register_projections(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(parse_query_shape, module)?)?; + module.add_function(wrap_pyfunction!(renames_explain_change, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs b/crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs new file mode 100644 index 0000000000..6df24cdd66 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs @@ -0,0 +1,43 @@ +//! Attach persisted migration identities through the existing cache boundary. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::native_host::models::ModelLookup; + +/// Attach persisted migration identities through the existing cache boundary. +pub fn with_migration_fingerprints<'py>( + entries: &Bound<'py, PyAny>, + models: &ModelLookup, + cache: &Bound<'py, PyAny>, + dialect: &Bound<'py, PyAny>, +) -> PyResult> { + use sqlbuild_python_base::boundary::main::call_with::call_with; + let py = entries.py(); + let mut attached: Vec> = Vec::new(); + for entry in entries.try_iter()? { + let entry = entry?; + let model = models.get(py, &native_field(&entry, "name")?.extract::()?)?; + let metadata = native_field(&entry, "fingerprint_metadata_json")?; + if model.is_none() || metadata.is_none() || native_field(&entry, "action")?.eq("skip")? { + attached.push(entry); + continue; + } + let fingerprint = call_with( + &native_field(cache, "fingerprint")?, + &[ + ("query_sql", query_sql(&model)?), + ("metadata_json", metadata), + ("ref_identities", PyDict::new(py).into_any()), + ("dialect", dialect.clone()), + ], + )?; + let kwargs = PyDict::new(py); + kwargs.set_item("migration_fingerprint", fingerprint)?; + attached.push(replaced_keywords(&entry, &kwargs)?); + } + cache.call_method0("persist")?; + PyTuple::new(py, attached) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/mod.rs new file mode 100644 index 0000000000..4dfc016833 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/mod.rs @@ -0,0 +1,7 @@ +//! Column and model migration planning: candidates, entries, overlays, handover and old names. + +pub(crate) mod _helpers; +pub mod constants; +pub(crate) mod errors; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs similarity index 67% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs rename to crates/sqlbuild-python-planner/src/planner_migrations/models.rs index 820ea2335c..f188887cb0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs @@ -1,143 +1,225 @@ -//! Read-only warehouse evidence gathered once for all model migrations in one plan. +//! Public records of planner migrations. -use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; -use std::collections::{HashMap, HashSet}; - -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyRef, PyRefMut, PyResult, Python, pyclass, pymethods, -}; -use pyo3::types::{PyBool, PyDict, PyString, PyTuple, PyTupleMethods}; +use crate::planner_migrations::_helpers::discovery::state_inspection::RelationKey; +use crate::planner_migrations::_helpers::discovery::state_inspection::lowered; +use crate::planner_migrations::_helpers::discovery::state_inspection::newest; +use crate::planner_migrations::_helpers::discovery::state_inspection::pending; +use crate::planner_migrations::_helpers::discovery::state_inspection::relation_key; +use crate::planner_migrations::_helpers::discovery::state_inspection::schema_or_empty; +use crate::planner_migrations::_helpers::discovery::state_inspection::tuple_of; +use crate::planner_migrations::constants::GET_COLUMNS_FOR_INSPECTION; +use crate::planner_migrations::constants::LIST_RELATIONS_FOR_INSPECTION; +use pyo3::prelude::Py; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyRef; +use pyo3::prelude::PyRefMut; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::prelude::pyclass; +use pyo3::prelude::pymethods; +use pyo3::prelude::{Bound, PyAny}; +use pyo3::types::PyBool; +use pyo3::types::PyDict; +use pyo3::types::PyString; +use pyo3::types::PyTuple; +use pyo3::types::PyTupleMethods; +use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_MODEL; use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_core::constants::MIGRATION_TABLE_NAME; use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; -use sqlbuild_state::events::main::newest_fingerprint::newest_fingerprint; -use sqlbuild_state::events::models::{EventRelation, FingerprintKey, FingerprintOrder}; - -use crate::bindings::_helpers::planner_migrations::constants::{ - GET_COLUMNS_FOR_INSPECTION, LIST_RELATIONS_FOR_INSPECTION, -}; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_MODEL; -use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; -use sqlbuild_python_front::state_rows::constants::{COLUMN_KIND, MODEL_KIND, OLD_NAME_KIND}; +use sqlbuild_python_front::state_rows::constants::COLUMN_KIND; +use sqlbuild_python_front::state_rows::constants::MODEL_KIND; +use sqlbuild_python_front::state_rows::constants::OLD_NAME_KIND; use sqlbuild_python_front::state_rows::main::migrations::layout::layout; use sqlbuild_python_front::state_rows::main::migrations::old_name_views::old_name_views; use sqlbuild_python_front::state_rows::main::migrations::read_events::read_events; use sqlbuild_python_front::state_rows::main::migrations::stored_old_name_columns::stored_old_name_columns; use sqlbuild_python_front::state_rows::models::StateAccess; +use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; +use sqlbuild_state::events::models::EventRelation; +use std::collections::HashMap; +use std::collections::HashSet; -/// A lowercased schema and relation name. -type RelationKey = (String, String); +#[derive(Debug)] +pub(crate) struct OldNamePlanner<'py> { + pub runtime: Bound<'py, PyAny>, + pub models: Vec>, + pub histories: Vec>, + pub views: Bound<'py, PyAny>, + pub columns: Bound<'py, PyAny>, + pub retention: Bound<'py, PyAny>, + pub now: Bound<'py, PyAny>, +} -/// Relations, fingerprints and migration events for one planning database. -#[pyclass(module = "sqlbuild._native", name = "MigrationStateInspection")] -pub(crate) struct MigrationStateInspection { - adapter: Py, - connection: Py, - database: Py, - known_fingerprints: Py, - read_schemas: HashSet, - /// Model fingerprints per lowercased schema, in first-read order. - fingerprints: Vec<(String, Vec>)>, - events: Vec>, - event_schemas: HashSet, - relations: HashMap>, - columns: HashMap>, - column_event_schemas: HashSet, - column_events: Vec>, - old_name_schemas: HashSet, - old_name_moves: Vec>, - old_name_facts: Vec>, +#[derive(Debug)] +pub(crate) struct ColumnCandidate<'py> { + pub model: Bound<'py, PyAny>, + pub warehouse: Bound<'py, PyDict>, + pub declared: Bound<'py, PyDict>, + pub fingerprint: Bound<'py, PyAny>, + pub previous: Bound<'py, PyAny>, + pub current: Bound<'py, PyAny>, } -fn lowered(value: &Bound<'_, PyAny>) -> PyResult { - value.call_method0("lower")?.extract() +#[derive(Debug)] +pub(crate) struct CheckedColumns<'py> { + pub entries: Vec>, + pub warnings: Vec>, } -fn schema_or_empty<'py>(value: Bound<'py, PyAny>) -> Bound<'py, PyAny> { - if value.is_none() { - PyString::new(value.py(), "").into_any() - } else { - value - } +/// The planning-phase inputs every migration planner reads. +#[derive(Debug)] +pub struct MigrationPhase<'py> { + pub runtime: Bound<'py, PyAny>, + pub scope: Bound<'py, PyAny>, + pub snapshot: Bound<'py, PyAny>, + pub overrides: Bound<'py, PyAny>, + pub deferral: Bound<'py, PyAny>, } -/// `((schema or "").lower(), name.lower())` of a relation or location. -fn relation_key(value: &Bound<'_, PyAny>) -> PyResult { - Ok(( - lowered(&schema_or_empty(native_field(value, "schema")?))?, - lowered(&native_field(value, "name")?)?, - )) +/// The column migration planning inputs beyond the phase. +#[derive(Debug)] +pub struct ColumnMigrationInputs<'py> { + pub full_refresh_model_names: Bound<'py, PyAny>, + pub physical_relations: Bound<'py, PyAny>, + pub source_columns: Bound<'py, PyAny>, } -/// The requested schemas not inspected yet, sorted, each with its lowercased key. -fn pending<'py>( - schemas: &Bound<'py, PyAny>, - inspected: &HashSet, -) -> PyResult, String)>> { - let mut pending: Vec<(String, Bound<'py, PyAny>, String)> = Vec::new(); - for schema in schemas.try_iter()? { - let schema: Bound<'py, PyAny> = schema?; - let key: String = lowered(&schema)?; - if !inspected.contains(&key) { - pending.push((schema.extract()?, schema, key)); - } - } - pending.sort_by(|left, right| left.0.cmp(&right.0)); - Ok(pending - .into_iter() - .map(|(_, schema, key)| (schema, key)) - .collect()) +/// The snapshot after model migrations overlay it, with the planned entries and their warnings. +#[derive(Debug)] +pub struct ModelMigrationPlanning<'py> { + pub snapshot: Bound<'py, PyAny>, + pub entries: Vec>, + pub warnings: Vec>, } -fn tuple_of<'py>(py: Python<'py>, items: &[Py]) -> PyResult> { - PyTuple::new(py, items.iter().map(|item| item.bind(py))) +/// The snapshot after column renames overlay it, with the planned entries, rename hints and warnings. +#[derive(Debug)] +pub struct ColumnMigrationPlanning<'py> { + pub snapshot: Bound<'py, PyAny>, + pub entries: Vec>, + pub hints: Vec>, + pub warnings: Vec>, } -fn fingerprint_key(fingerprint: &Bound<'_, PyAny>) -> PyResult { - let target_name: Bound<'_, PyAny> = native_field(fingerprint, "target_name")?; - let target: Option = if target_name.is_none() { - None - } else { - Some(EventRelation { - database: native_field(fingerprint, "target_database")?.extract()?, - schema: native_field(fingerprint, "target_schema")?.extract()?, - name: target_name.extract()?, - }) - }; - Ok(FingerprintKey { - order: FingerprintOrder { - timestamp: native_field(fingerprint, "ts")? - .call_method0("isoformat")? - .extract()?, - run_id: native_field(fingerprint, "run_id")?.extract()?, - }, - node_name: native_field(fingerprint, "node_name")?.extract()?, - target, - }) +/// The old-name view steps, the recorded live views and the warnings they raise. +#[derive(Debug)] +pub struct OldNameViewPlanning<'py> { + pub entries: Bound<'py, PyAny>, + pub views: Bound<'py, PyAny>, + pub warnings: Vec>, } -fn newest<'py>( - py: Python<'py>, - candidates: &[Py], - (target, model_name): (Option<&EventRelation>, Option<&str>), -) -> PyResult>> { - let keys: Vec = candidates - .iter() - .map(|candidate| fingerprint_key(candidate.bind(py))) - .collect::>()?; - Ok(newest_fingerprint(&keys, target, model_name) - .map(|index| candidates[index].bind(py).clone())) +/// One destination model whose data should come from an earlier relation. +#[derive(Debug, Clone)] +pub(crate) struct ModelMigrationRequest<'py> { + pub model: Bound<'py, PyAny>, + pub discovery: Bound<'py, PyAny>, + pub raw_origin: Option>, + pub origin_location: Option>, + pub origin_model: Bound<'py, PyAny>, + pub force: bool, + pub identity_only: bool, +} + +/// One project model's explicit migrate_from, resolved to a relation. +#[derive(Debug)] +pub(crate) struct ModelMigrationDeclaration<'py> { + pub model_name: Bound<'py, PyAny>, + pub origin_location: Bound<'py, PyAny>, + pub origin_model: Bound<'py, PyAny>, +} + +/// Manual and automatically discovered migration requests for one plan. +#[derive(Debug)] +pub(crate) struct ModelMigrationDiscovery<'py> { + pub requests: Vec>, + pub warnings: Vec>, + pub destination_fingerprints: Bound<'py, PyDict>, +} + +/// Compatibility status plus human-readable blocking findings. +#[derive(Debug)] +pub(crate) struct MigrationCompatibilityResult<'py> { + pub status: Bound<'py, PyAny>, + pub findings: Bound<'py, PyAny>, +} + +/// The fingerprint inputs shared by migration matching and destination fingerprints. +#[derive(Debug)] +pub(crate) struct FingerprintInputs<'py> { + pub metadata_jsons: Bound<'py, PyAny>, + pub dialect: Bound<'py, PyAny>, + pub fingerprints: Bound<'py, PyAny>, +} + +/// One resolved migration request decided against live state. +#[derive(Debug)] +pub(crate) struct MigrationDecisionRequest<'a, 'py> { + pub runtime: Bound<'py, PyAny>, + pub request: &'a ModelMigrationRequest<'py>, + pub origin: Bound<'py, PyAny>, + pub origin_model: Bound<'py, PyAny>, + pub snapshot: Bound<'py, PyAny>, + pub state: Bound<'py, PyAny>, + pub snapshots_config: Bound<'py, PyAny>, +} + +/// What checking an origin relation's compatibility with its destination model reads. +#[derive(Debug)] +pub(crate) struct CompatibilityCheck<'py> { + pub model: Bound<'py, PyAny>, + pub origin_relation: Bound<'py, PyAny>, + pub origin_columns: Bound<'py, PyAny>, + pub sql_analysis_enabled: Bound<'py, PyAny>, + pub column_dialect: Bound<'py, PyAny>, + pub snapshots_config: Bound<'py, PyAny>, +} + +/// What discovering automatic and manual migration requests reads. +#[derive(Debug)] +pub(crate) struct DiscoveryRequest<'py> { + pub runtime: Bound<'py, PyAny>, + pub scope: Bound<'py, PyAny>, + pub snapshot: Bound<'py, PyAny>, + pub manual_requests: Vec>, + pub declarations: Vec>, + pub state: Bound<'py, PyAny>, + pub project_schemas: Bound<'py, PyAny>, + pub fingerprints: Bound<'py, PyAny>, +} + +/// Relations, fingerprints and migration events for one planning database. +#[pyclass(module = "sqlbuild._native", name = "MigrationStateInspection")] +#[derive(Debug)] +pub struct MigrationStateInspection { + pub(crate) adapter: Py, + pub(crate) connection: Py, + pub(crate) database: Py, + pub(crate) known_fingerprints: Py, + pub(crate) read_schemas: HashSet, + /// Model fingerprints per lowercased schema, in first-read order. + pub(crate) fingerprints: Vec<(String, Vec>)>, + pub(crate) events: Vec>, + pub(crate) event_schemas: HashSet, + pub(crate) relations: HashMap>, + pub(crate) columns: HashMap>, + pub(crate) column_event_schemas: HashSet, + pub(crate) column_events: Vec>, + pub(crate) old_name_schemas: HashSet, + pub(crate) old_name_moves: Vec>, + pub(crate) old_name_facts: Vec>, } impl MigrationStateInspection { - fn access<'py>(&self, py: Python<'py>) -> PyResult> { + pub(crate) fn access<'py>(&self, py: Python<'py>) -> PyResult> { let adapter: &Bound<'py, PyAny> = self.adapter.bind(py); Ok(StateAccess { connection: self.connection.bind(py).clone(), @@ -146,7 +228,7 @@ impl MigrationStateInspection { }) } - fn list_relations<'py>( + pub(crate) fn list_relations<'py>( &self, py: Python<'py>, schemas: Vec>, @@ -166,7 +248,7 @@ impl MigrationStateInspection { .collect() } - fn model_fingerprints<'py>( + pub(crate) fn model_fingerprints<'py>( &self, py: Python<'py>, schema: &Bound<'py, PyAny>, @@ -218,7 +300,7 @@ impl MigrationStateInspection { impl MigrationStateInspection { #[new] #[pyo3(signature = (*, adapter, connection, database, known_fingerprints = None))] - fn new( + pub(crate) fn new( adapter: &Bound<'_, PyAny>, connection: &Bound<'_, PyAny>, database: &Bound<'_, PyAny>, @@ -255,25 +337,25 @@ impl MigrationStateInspection { /// The database whose namespaces this inspection reads. #[getter] - pub(crate) fn database(&self, py: Python<'_>) -> Py { + pub fn database(&self, py: Python<'_>) -> Py { self.database.clone_ref(py) } /// Every migration event read so far. #[getter] - pub(crate) fn events<'py>(&self, py: Python<'py>) -> PyResult> { + pub fn events<'py>(&self, py: Python<'py>) -> PyResult> { tuple_of(py, &self.events) } /// Every column migration event read so far. #[getter] - pub(crate) fn column_events<'py>(&self, py: Python<'py>) -> PyResult> { + pub fn column_events<'py>(&self, py: Python<'py>) -> PyResult> { tuple_of(py, &self.column_events) } /// The old-name history of every move read by `inspect_old_name_views`. #[getter] - pub(crate) fn old_name_histories<'py>(&self, py: Python<'py>) -> PyResult> { + pub fn old_name_histories<'py>(&self, py: Python<'py>) -> PyResult> { old_name_views( tuple_of(py, &self.old_name_moves)?.as_any(), tuple_of(py, &self.old_name_facts)?.as_any(), @@ -282,7 +364,10 @@ impl MigrationStateInspection { /// Read the column migration state table once for every schema not inspected yet. #[pyo3(signature = (*, schemas))] - fn inspect_column_events(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + pub(crate) fn inspect_column_events( + slf: &Bound<'_, Self>, + schemas: &Bound<'_, PyAny>, + ) -> PyResult<()> { let py: Python<'_> = slf.py(); let pending = pending(schemas, &slf.borrow().column_event_schemas)?; if pending.is_empty() { @@ -322,7 +407,10 @@ impl MigrationStateInspection { /// Read moves and old-name facts once for every schema not inspected yet. #[pyo3(signature = (*, schemas))] - fn inspect_old_name_views(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + pub(crate) fn inspect_old_name_views( + slf: &Bound<'_, Self>, + schemas: &Bound<'_, PyAny>, + ) -> PyResult<()> { let py: Python<'_> = slf.py(); let pending = pending(schemas, &slf.borrow().old_name_schemas)?; if pending.is_empty() { @@ -376,7 +464,10 @@ impl MigrationStateInspection { /// Read migration events once from schemas known to hold the migration table. #[pyo3(signature = (*, schemas))] - fn inspect_migration_events(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + pub(crate) fn inspect_migration_events( + slf: &Bound<'_, Self>, + schemas: &Bound<'_, PyAny>, + ) -> PyResult<()> { let py: Python<'_> = slf.py(); let mut sorted: Vec<(String, Bound<'_, PyAny>)> = Vec::new(); for schema in schemas.try_iter()? { @@ -403,7 +494,10 @@ impl MigrationStateInspection { /// Read state tables once for every schema not inspected yet. #[pyo3(signature = (*, schemas))] - fn inspect_schemas(slf: &Bound<'_, Self>, schemas: &Bound<'_, PyAny>) -> PyResult<()> { + pub(crate) fn inspect_schemas( + slf: &Bound<'_, Self>, + schemas: &Bound<'_, PyAny>, + ) -> PyResult<()> { let py: Python<'_> = slf.py(); let pending = pending(schemas, &slf.borrow().read_schemas)?; if pending.is_empty() { @@ -442,7 +536,10 @@ impl MigrationStateInspection { /// List candidate relations and their columns with one metadata read. #[pyo3(signature = (*, locations))] - fn inspect_relations(slf: &Bound<'_, Self>, locations: &Bound<'_, PyAny>) -> PyResult<()> { + pub(crate) fn inspect_relations( + slf: &Bound<'_, Self>, + locations: &Bound<'_, PyAny>, + ) -> PyResult<()> { let py: Python<'_> = slf.py(); let mut wanted: HashSet = HashSet::new(); let mut schemas: Vec = Vec::new(); @@ -513,7 +610,11 @@ impl MigrationStateInspection { } /// The listed relation for a location. - fn relation(&self, py: Python<'_>, location: &Bound<'_, PyAny>) -> PyResult> { + pub(crate) fn relation( + &self, + py: Python<'_>, + location: &Bound<'_, PyAny>, + ) -> PyResult> { Ok(match self.relations.get(&relation_key(location)?) { Some(relation) => relation.clone_ref(py), None => py.None(), @@ -521,7 +622,11 @@ impl MigrationStateInspection { } /// Listed columns for a location. - fn relation_columns(&self, py: Python<'_>, location: &Bound<'_, PyAny>) -> PyResult> { + pub(crate) fn relation_columns( + &self, + py: Python<'_>, + location: &Bound<'_, PyAny>, + ) -> PyResult> { Ok(match self.columns.get(&relation_key(location)?) { Some(columns) => columns.clone_ref(py), None => PyTuple::empty(py).into_any().unbind(), @@ -530,7 +635,7 @@ impl MigrationStateInspection { /// The latest model fingerprint that built the location or named model. #[pyo3(signature = (*, location, model_name))] - fn model_fingerprint<'py>( + pub(crate) fn model_fingerprint<'py>( &self, py: Python<'py>, location: &Bound<'py, PyAny>, @@ -551,7 +656,7 @@ impl MigrationStateInspection { /// The newest model fingerprint with this node name in any inspected schema. #[pyo3(signature = (*, model_name))] - fn named_fingerprint<'py>( + pub(crate) fn named_fingerprint<'py>( &self, py: Python<'py>, model_name: &str, @@ -565,7 +670,7 @@ impl MigrationStateInspection { /// Latest model fingerprints whose names are no longer in the project. #[pyo3(signature = (*, project_model_names))] - fn orphan_fingerprints<'py>( + pub(crate) fn orphan_fingerprints<'py>( &self, py: Python<'py>, project_model_names: &Bound<'py, PyAny>, @@ -582,17 +687,3 @@ impl MigrationStateInspection { PyTuple::new(py, orphans) } } - -/// A fresh live-state inspection over the runtime's connection and planning database. -pub(super) fn inspection<'py>( - runtime: &Bound<'py, PyAny>, - known_fingerprints: Option<&Bound<'py, PyAny>>, -) -> PyResult> { - let state = MigrationStateInspection::new( - &runtime_field(runtime, |runtime| &runtime.adapter)?, - &runtime_field(runtime, |runtime| &runtime.connection)?, - &super::planning_origin::planning_database(runtime)?, - known_fingerprints, - )?; - Ok(Bound::new(runtime.py(), state)?.into_any()) -} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/contexts.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/contexts.rs new file mode 100644 index 0000000000..68a9ee4db6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/contexts.rs @@ -0,0 +1,30 @@ +//! Planner scope and identity contexts: built from keywords and read by planner phases. + +use pyo3::prelude::{Bound, Py, PyAny, PyResult}; +use pyo3::pymethods; +use pyo3::{PyClass, pyclass::boolean_struct::True}; + +use crate::planner_phases::models::PlannerIdentityContext; + +#[pymethods] +impl PlannerIdentityContext { + #[new] + #[pyo3(signature = (*, version_identities, stale_warning_identities))] + fn new(version_identities: Py, stale_warning_identities: Py) -> Self { + Self { + version_identities, + stale_warning_identities, + } + } +} + +/// One field of a frozen native planner class passed as a Python object. +pub(crate) fn class_field<'py, T>( + value: &Bound<'py, PyAny>, + field: fn(&T) -> &Py, +) -> PyResult> +where + T: PyClass + Sync, +{ + Ok(field(value.cast::()?.get()).bind(value.py()).clone()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs similarity index 75% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs index ca87274851..48d89d4596 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs @@ -3,38 +3,13 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::planner_phases::models::{ - PlannerIdentityContext, PlannerRuntime, PlannerScopeResolution, PlannerWarehouseState, +use crate::planner_phases::models::{ + PlannerIdentityContext, PlannerRuntime, PlannerWarehouseState, }; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use sqlbuild_python_project::planner_scope::models::RetainedScope; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[ - GetterEntry { - class: class_of::, - read: read_retained_scope, - }, - GetterEntry { - class: class_of::, - read: read_planner_identity_context, - }, - GetterEntry { - class: class_of::, - read: read_planner_runtime, - }, - GetterEntry { - class: class_of::, - read: read_planner_scope_resolution, - }, - GetterEntry { - class: class_of::, - read: read_planner_warehouse_state, - }, -]; - -fn read_retained_scope<'py>( +pub(crate) fn read_retained_scope<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -44,7 +19,7 @@ fn read_retained_scope<'py>( Some(cell.get().data.field(value.py(), name)) } -fn read_planner_identity_context<'py>( +pub(crate) fn read_planner_identity_context<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -62,7 +37,7 @@ fn read_planner_identity_context<'py>( }) } -fn read_planner_runtime<'py>( +pub(crate) fn read_planner_runtime<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -85,7 +60,7 @@ fn read_planner_runtime<'py>( }) } -fn read_planner_scope_resolution<'py>( +pub(crate) fn read_planner_scope_resolution<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -104,7 +79,7 @@ fn read_planner_scope_resolution<'py>( }) } -fn read_planner_warehouse_state<'py>( +pub(crate) fn read_planner_warehouse_state<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/mod.rs new file mode 100644 index 0000000000..c7081fc170 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod contexts; +pub(crate) mod getters; +pub(crate) mod runtime; +pub(crate) mod warehouse_state; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs index 2439da6be4..5b3535f9a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs @@ -1,11 +1,11 @@ //! The planner runtime and warehouse state: built from keyword arguments and read by planner phases. -use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; +use crate::planner_warehouse::models::MigrationFingerprintCache; use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; use pyo3::types::{PyDict, PyTuple}; -use crate::bindings::_helpers::planner_phases::models::{PlannerRuntime, PlannerWarehouseState}; +use crate::planner_phases::models::{PlannerRuntime, PlannerWarehouseState}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::build_command::main::utc_now::utc_now; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs similarity index 86% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs index a682495b34..422b8bdbc9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs @@ -1,20 +1,18 @@ //! The warehouse inspection phase: snapshot, migrations and inspection relations in one pass. -use crate::bindings::_helpers::planner_entries::models::RelationsRequest; -use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; -use crate::bindings::_helpers::planner_migrations::column_planning::plan_column_migrations; -use crate::bindings::_helpers::planner_migrations::model_planning::plan_model_migrations; -use crate::bindings::_helpers::planner_migrations::models::{ - ColumnMigrationInputs, MigrationPhase, -}; -use crate::bindings::_helpers::planner_migrations::old_name_planning::plan_old_name_views as plan_old_name_migration_views; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; -use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; -use crate::bindings::_helpers::planner_phases::models::PlannerWarehouseState; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; -use crate::bindings::_helpers::planner_warehouse::snapshot::{SnapshotRequest, warehouse_snapshot}; +use crate::planner_entries::main::entries::planner_relations_context::planner_relations_context; +use crate::planner_entries::models::RelationsRequest; +use crate::planner_migrations::main::plan_column_migrations::plan_column_migrations; +use crate::planner_migrations::main::plan_model_migrations::plan_model_migrations; +use crate::planner_migrations::main::plan_old_name_views::plan_old_name_views as plan_old_name_migration_views; +use crate::planner_migrations::models::{ColumnMigrationInputs, MigrationPhase}; +use crate::planner_phases::_helpers::inputs::contexts::class_field; +use crate::planner_phases::_helpers::inputs::runtime::runtime_field; +use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; +use crate::planner_phases::models::PlannerWarehouseState; +use crate::planner_warehouse::main::inspection::warehouse_snapshot::warehouse_snapshot; +use crate::planner_warehouse::models::MigrationFingerprintCache; +use crate::planner_warehouse::models::SnapshotRequest; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -22,6 +20,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_project::graph::main::project::compiled_graph_keys::compiled_graph_keys; use sqlbuild_python_project::planner_phases::main::native::native; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use std::time::Instant; fn progress(runtime: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/mod.rs new file mode 100644 index 0000000000..68672218c6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod inputs; +pub(crate) mod outputs; +pub(crate) mod rebuilds; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs index 9dbc69077b..76c81a61b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/buildability.rs @@ -1,11 +1,10 @@ //! Selected-scope buildability validation before change planning. -use crate::bindings::_helpers::planner_phases::constants::MISSING_UPSTREAM_CODE; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; -use crate::bindings::_helpers::planner_warehouse::source_tables::check_selected_source_tables_exist; +use crate::planner_phases::_helpers::inputs::contexts::class_field; +use crate::planner_phases::_helpers::inputs::runtime::runtime_field; +use crate::planner_phases::_helpers::inputs::runtime::warehouse_field; +use crate::planner_phases::constants::MISSING_UPSTREAM_CODE; +use crate::planner_warehouse::main::sources::check_selected_source_tables_exist::check_selected_source_tables_exist; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message; use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; @@ -16,6 +15,7 @@ use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::planner_identity::main::execution_origins::execution_origins; use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use sqlbuild_python_project::resource_views::main::seed_reads::external::external; use sqlbuild_python_project::resource_views::main::seed_reads::key::key; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs similarity index 80% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs index 19ae42d660..5e5c90a152 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs @@ -1,11 +1,10 @@ //! Expected version identities and stale-warning change detection. -use crate::bindings::_helpers::planner_changes::detect::scope_model_changes; -use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::models::IdentityWork; -use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; -use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; +use crate::planner_changes::main::scope_model_changes::scope_model_changes; +use crate::planner_identity::main::direct_model_version_identities::direct_model_version_identities; +use crate::planner_phases::_helpers::inputs::contexts::class_field; +use crate::planner_phases::models::IdentityWork; +use crate::planner_phases::models::PlannerIdentityContext; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::sync::PyOnceLock; @@ -21,6 +20,7 @@ use sqlbuild_python_project::compiled_views::main::project::functions::functions use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; use sqlbuild_python_project::planner_phases::main::native::native; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; use std::collections::{BTreeMap, HashSet}; static IDENTITY_WORK: PyOnceLock> = PyOnceLock::new(); @@ -47,7 +47,7 @@ pub(crate) fn planner_identity_work_info( Ok(info) } -fn version_identities<'py>( +pub(crate) fn version_identities<'py>( project: &Bound<'py, PyAny>, scope: Bound<'py, PyAny>, recorded_fingerprints: &Bound<'py, PyAny>, @@ -65,36 +65,6 @@ fn version_identities<'py>( ) } -pub(crate) fn build_planner_identity_context<'py>( - project: &Bound<'py, PyAny>, - (scopes, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - include_stale_warning_identities: bool, -) -> PyResult> { - let recorded_fingerprints: Bound<'py, PyAny> = native_field(snapshot, "fingerprints")?; - let identities = version_identities( - project, - class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, - &recorded_fingerprints, - )?; - let stale = if include_stale_warning_identities { - version_identities( - project, - class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)?, - &recorded_fingerprints, - )? - } else { - identities.clone() - }; - Ok(Bound::new( - project.py(), - PlannerIdentityContext { - version_identities: identities.unbind(), - stale_warning_identities: stale.unbind(), - }, - )? - .into_any()) -} - fn reusable_model_names( identities: &Bound<'_, PyAny>, execution_changes: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs new file mode 100644 index 0000000000..fcf2752f38 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod buildability; +pub(crate) mod identities; +pub(crate) mod output_assembly; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs similarity index 72% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs index 26493cd00c..f3c26b7b1f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs @@ -1,25 +1,22 @@ //! Plan output assembly: storage policies, warnings and direct-mode metadata. -use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; -use crate::bindings::_helpers::planner_entries::plan_output::plan_output_fields; -use crate::bindings::_helpers::planner_identity::staleness::build_stale_out_of_selection_warnings; -use crate::bindings::_helpers::planner_migrations::fingerprint::with_migration_fingerprints; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; -use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; -use crate::bindings::_helpers::planner_phases::models::{ +use crate::planner_entries::main::plans::plan_output_fields::plan_output_fields; +use crate::planner_entries::models::PlanOutputInputs; +use crate::planner_identity::main::build_stale_out_of_selection_warnings::build_stale_out_of_selection_warnings; +use crate::planner_migrations::main::with_migration_fingerprints::with_migration_fingerprints; +use crate::planner_phases::_helpers::inputs::contexts::class_field; +use crate::planner_phases::_helpers::inputs::runtime::runtime_field; +use crate::planner_phases::_helpers::inputs::runtime::warehouse_field; +use crate::planner_phases::_helpers::rebuilds::retention::{plan_retention, plan_table_types}; +use crate::planner_phases::models::PlannerIdentityContext; +use crate::planner_phases::models::{ PlanOutputContext, PlanOutputFields, PlanWarningContext, PlannedModelEntries, StoragePolicyContext, }; -use crate::bindings::_helpers::planner_phases::retention::{plan_retention, plan_table_types}; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; -use crate::bindings::_helpers::planner_warehouse::source_freshness::source_freshness_unknown_warnings; +use crate::planner_warehouse::main::sources::source_freshness_unknown_warnings::source_freshness_unknown_warnings; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; -use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; @@ -30,11 +27,14 @@ use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_phases::main::replace::replace; use sqlbuild_python_project::planner_phases::main::sorted_names::sorted_names; use sqlbuild_python_project::planner_phases::main::strings::strings; -use sqlbuild_python_project::planner_phases::main::text::text; use sqlbuild_python_project::planner_phases::main::tuple::tuple; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; -fn concat<'py>(py: Python<'py>, parts: &[Bound<'py, PyAny>]) -> PyResult> { +pub(crate) fn concat<'py>( + py: Python<'py>, + parts: &[Bound<'py, PyAny>], +) -> PyResult> { let mut items: Vec<_> = Vec::new(); for part in parts { for item in part.try_iter()? { @@ -45,7 +45,7 @@ fn concat<'py>(py: Python<'py>, parts: &[Bound<'py, PyAny>]) -> PyResult PlanOutputFields<'py> { - fn field(&self, name: &str) -> PyResult> { + pub(crate) fn field(&self, name: &str) -> PyResult> { self.fields .iter() .find(|(field, _)| *field == name) @@ -53,7 +53,7 @@ impl<'py> PlanOutputFields<'py> { .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(name.to_owned())) } - fn set(&mut self, name: &'static str, value: Bound<'py, PyAny>) { + pub(crate) fn set(&mut self, name: &'static str, value: Bound<'py, PyAny>) { match self.fields.iter_mut().find(|(field, _)| *field == name) { Some(existing) => existing.1 = value, None => self.fields.push((name, value)), @@ -285,85 +285,6 @@ pub(crate) fn add_plan_warnings<'py>( Ok(output) } -/// The unselected models the selected scope reads, sorted. -pub(crate) fn unselected_upstream_models(scopes: &Bound<'_, PyAny>) -> PyResult> { - let selected = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; - let selected = sqlbuild_python_project::planner_scope::main::scope_data::scope_data(&selected)?; - let mut names: Vec = Vec::new(); - for (key, deps) in &selected.upstream { - if !selected.selected.contains(key) { - continue; - } - for dep in deps { - if dep.0 == MODEL && !selected.selected.contains(dep) { - names.push(dep.1.clone()); - } - } - } - names.sort(); - names.dedup(); - Ok(names) -} - -/// Note unselected upstreams deferred to pre-migration names, and each lookup in `failures`. -pub(crate) fn add_deferred_origin_notes<'py>( - mut output: PlanOutputFields<'py>, - (project, scopes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - (deferral, failures): (&Bound<'py, PyAny>, &[(String, String)]), -) -> PyResult> { - let py = project.py(); - let locations = native_field(deferral, "deferred_locations")?; - if locations.is_none() { - return Ok(output); - } - let names: Vec = unselected_upstream_models(scopes)?; - let warning = |name: &str, message: &str| -> PyResult> { - native( - py, - RecordId::PlanWarning, - &[ - ("model_name", text(py, name)), - ( - "severity", - enum_member(py, RecordId::WarningSeverity, "WARNING")?, - ), - ("message", text(py, message)), - ], - ) - }; - let mut notes: Vec> = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { - let model = model?; - let name: String = native_field(&model, "name")?.extract()?; - if names.binary_search(&name).is_err() { - continue; - } - let Some(location) = locations.downcast::()?.get_item(&name)? else { - continue; - }; - if native_field(&location, "name")? - .eq(native_field(&native_field(&model, "destination")?, "name")?)? - { - continue; - } - notes.push(warning( - &name, - &format!( - "deferred to pre-migration name {}: the deferred target has not built model '{name}' under its current name yet", - native_field(&location, "qualified_name")? - ), - )?); - } - for (name, message) in failures { - notes.push(warning(name, message)?); - } - if !notes.is_empty() { - let warnings = concat(py, &[output.field("warnings")?, tuple(py, notes)?])?; - output.set("warnings", warnings); - } - Ok(output) -} - fn source_names<'py>(items: &Bound<'py, PyAny>) -> PyResult> { let mut names: Vec<_> = Vec::new(); for item in items.try_iter()? { @@ -383,7 +304,9 @@ fn age_names<'py>(statuses: &Bound<'py, PyAny>, wanted: &str) -> PyResult(source_freshness: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn freshness_metadata<'py>( + source_freshness: &Bound<'py, PyAny>, +) -> PyResult> { let py = source_freshness.py(); let propagation = native_field(source_freshness, "propagation")?; let propagated = |field: &str| -> PyResult> { @@ -435,27 +358,3 @@ fn freshness_metadata<'py>(source_freshness: &Bound<'py, PyAny>) -> PyResult( - mut output: PlanOutputFields<'py>, - source_freshness: &Bound<'py, PyAny>, - policies: &Bound<'py, PyAny>, -) -> PyResult> { - let py = source_freshness.py(); - let diagnostics = PyDict::new(py); - diagnostics.set_item("mode", text(py, "direct"))?; - diagnostics.set_item("enabled", native_field(policies, "selection_diagnostics")?)?; - output.metadata.push(( - "direct_source_freshness", - freshness_metadata(source_freshness)?, - )); - output.metadata.push(( - "direct_remaining_stale_model_names", - sorted_names(py, Vec::new())?, - )); - output - .metadata - .push(("selection_diagnostics", diagnostics.into_any())); - Ok(output) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/deferred_origins.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/deferred_origins.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/deferred_origins.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/deferred_origins.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs similarity index 71% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs index 6cef425326..582a32a21f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs @@ -1,56 +1,17 @@ //! Write-hash reconciliation and plan entry construction. -use crate::bindings::_helpers::planner_entries::contexts::default_microbatch_limit_action; -use crate::bindings::_helpers::planner_entries::model_entries::plan_entries; -use crate::bindings::_helpers::planner_entries::models::{ - PlanEntriesRequest, PlanEntryBuildInputs, RelationsRequest, -}; -use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; -use crate::bindings::_helpers::planner_identity::identities::merge_recomputed_model_changes; -use crate::bindings::_helpers::planner_identity::identities::with_honest_model_write_hashes; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; -use crate::bindings::_helpers::planner_phases::models::{ChangeReconciliation, ModelEntryResults}; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; +use crate::planner_entries::main::entries::default_microbatch_limit_action::default_microbatch_limit_action; +use crate::planner_entries::main::entries::plan_entries::plan_entries; +use crate::planner_entries::main::entries::planner_relations_context::planner_relations_context; +use crate::planner_entries::models::{PlanEntriesRequest, PlanEntryBuildInputs, RelationsRequest}; +use crate::planner_phases::_helpers::inputs::runtime::runtime_field; +use crate::planner_phases::_helpers::inputs::runtime::warehouse_field; +use crate::planner_phases::models::{ChangeReconciliation, ModelEntryResults}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyFrozenSet; -use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::command_runtime::main::resolve_configured_concurrency::resolve_configured_concurrency; -/// The execution scope's changes with honest write hashes, and the actions merged from them. -pub(crate) fn reconcile_execution_changes<'py>( - (execution_scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - identities: &Bound<'py, PyAny>, - resolved_actions: &Bound<'py, PyAny>, - changes: &Bound<'py, PyAny>, -) -> PyResult> { - let py = execution_scope.py(); - let honest = with_honest_model_write_hashes( - py, - Some(keywords( - py, - &[ - ("scope", execution_scope.clone()), - ("snapshot", snapshot.clone()), - ("changes", changes.clone()), - ( - "version_identities", - class_field(identities, |i: &PlannerIdentityContext| { - &i.version_identities - })?, - ), - ], - )?), - )?; - let resolved = merge_recomputed_model_changes(resolved_actions, &honest)?; - Ok(ChangeReconciliation { - changes: honest, - resolved_actions: resolved, - }) -} - fn config_or<'py>( config: &Bound<'py, PyAny>, path: &[&str], diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/full_refresh.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/full_refresh.rs new file mode 100644 index 0000000000..dd3b389340 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/full_refresh.rs @@ -0,0 +1,13 @@ +//! Per-model full-refresh resolution and full-rebuild refusals. + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; + +const FULL_REFRESH_KEY: &str = "full_refresh"; + +pub(crate) fn configured_full_refresh<'py>( + model: &Bound<'py, PyAny>, +) -> PyResult> { + config_value(model.py(), &model_values(model)?, FULL_REFRESH_KEY) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/mod.rs new file mode 100644 index 0000000000..23c86a7a82 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod deferred_origins; +pub(crate) mod execution_changes; +pub(crate) mod full_refresh; +pub(crate) mod retention; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs index 0db0b3022e..14cb257a33 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs @@ -1,13 +1,14 @@ //! Warehouse-aware time-travel retention and Snowflake table-type planning. +use crate::planner_phases::main::rebuilds::effective_target_config::effective_target_config; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::planner_phases::constants::{ +use crate::planner_phases::_helpers::inputs::runtime::runtime_field; +use crate::planner_phases::constants::{ BIGQUERY_ADAPTER, DECREASE_WARNING, DOWNGRADE_HISTORY_DAYS, PERMANENT_TABLE_TYPE, RETENTION_DECREASE, RETENTION_INCREASE, RETENTION_MATCH, SNOWFLAKE_ADAPTER, TRANSIENT_TABLE_TYPE, }; -use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -21,7 +22,6 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; -use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::typed_values::typed_values; use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; @@ -29,7 +29,6 @@ use sqlbuild_python_project::compiled_views::main::compiled_models::name_object: use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::planner_phases::main::construct::construct; use sqlbuild_python_project::planner_phases::main::member::member; -use sqlbuild_python_project::planner_phases::main::native::native; use sqlbuild_python_project::planner_phases::main::native_member::native_member; use sqlbuild_python_project::planner_phases::main::optional_text::optional_text; use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; @@ -43,20 +42,6 @@ use std::collections::{BTreeMap, BTreeSet}; const MATERIALIZED_KEY: &str = "materialized"; -/// The resolved config of the planned target, or defaults without one. -pub(crate) fn effective_target_config<'py>( - runtime: &Bound<'py, PyAny>, -) -> PyResult> { - let py = runtime.py(); - let project_config = runtime_field(runtime, |runtime| &runtime.project_config)?; - let local_config = runtime_field(runtime, |runtime| &runtime.local_config)?; - let target_name = native_field(&native_field(runtime, "project")?, "effective_target_name")?; - if project_config.is_none() || local_config.is_none() || target_name.is_none() { - return native(py, RecordId::TargetConfig, &[]); - } - resolve_target_config(py, &project_config, &local_config, &target_name) -} - /// A deterministic identifier-fitted conversion copy name. #[pyfunction] #[pyo3(signature = (*, target_name, identifier_limit))] diff --git a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs new file mode 100644 index 0000000000..e53f254daf --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs @@ -0,0 +1,61 @@ +//! Public constants of planner phases. + +use crate::planner_phases::_helpers::inputs::getters::read_planner_identity_context; +use crate::planner_phases::_helpers::inputs::getters::read_planner_runtime; +use crate::planner_phases::_helpers::inputs::getters::read_planner_scope_resolution; +use crate::planner_phases::_helpers::inputs::getters::read_planner_warehouse_state; +use crate::planner_phases::_helpers::inputs::getters::read_retained_scope; +use crate::planner_phases::models::PlannerIdentityContext; +use crate::planner_phases::models::PlannerRuntime; +use crate::planner_phases::models::PlannerWarehouseState; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; +use sqlbuild_python_project::planner_scope::models::RetainedScope; + +pub(crate) const SNOWFLAKE_ADAPTER: &str = "snowflake"; + +pub(crate) const BIGQUERY_ADAPTER: &str = "bigquery"; + +pub(crate) const TRANSIENT_TABLE_TYPE: &str = "transient"; + +pub(crate) const PERMANENT_TABLE_TYPE: &str = "permanent"; + +pub(crate) const DECREASE_WARNING: &str = + "Retention decreases are irreversible once warehouse history expires."; + +pub(crate) const DOWNGRADE_HISTORY_DAYS: i64 = 90; + +pub(crate) const FULL_REFRESH_DISABLED_REBUILD_CODE: &str = "S203"; + +pub(crate) const MISSING_UPSTREAM_CODE: &str = "S301"; + +pub(crate) const RETENTION_DECREASE: &str = "decrease"; + +pub(crate) const RETENTION_INCREASE: &str = "increase"; + +pub(crate) const RETENTION_MATCH: &str = "match"; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_retained_scope, + }, + GetterEntry { + class: class_of::, + read: read_planner_identity_context, + }, + GetterEntry { + class: class_of::, + read: read_planner_runtime, + }, + GetterEntry { + class: class_of::, + read: read_planner_scope_resolution, + }, + GetterEntry { + class: class_of::, + read: read_planner_warehouse_state, + }, +]; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/class_field.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/class_field.rs new file mode 100644 index 0000000000..15dda60248 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/class_field.rs @@ -0,0 +1,18 @@ +//! One field of a frozen native planner class passed as a Python object. + +use crate::planner_phases::_helpers::inputs::contexts; +use pyo3::PyClass; +use pyo3::prelude::{Bound, Py, PyAny, PyResult}; +use pyo3::pyclass::boolean_struct::True; + +/// One field of a frozen native planner class passed as a Python object. +#[inline] +pub fn class_field<'py, T>( + value: &Bound<'py, PyAny>, + field: fn(&T) -> &Py, +) -> PyResult> +where + T: PyClass + Sync, +{ + contexts::class_field(value, field) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/gather_planner_warehouse_state.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/gather_planner_warehouse_state.rs new file mode 100644 index 0000000000..761cf17497 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/gather_planner_warehouse_state.rs @@ -0,0 +1,15 @@ +//! Gather planner warehouse state. + +use crate::planner_phases::_helpers::inputs::warehouse_state; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Gather planner warehouse state. +#[inline] +pub fn gather_planner_warehouse_state<'py>( + runtime: &Bound<'py, PyAny>, + scopes: &Bound<'py, PyAny>, + overrides: &Bound<'py, PyAny>, + deferral: &Bound<'py, PyAny>, +) -> PyResult> { + warehouse_state::gather_planner_warehouse_state(runtime, scopes, overrides, deferral) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/mod.rs new file mode 100644 index 0000000000..b02e5cc9c9 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/mod.rs @@ -0,0 +1,4 @@ +pub mod class_field; +pub mod gather_planner_warehouse_state; +pub mod runtime_field; +pub mod warehouse_field; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_field.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_field.rs new file mode 100644 index 0000000000..f9f1f658c4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/runtime_field.rs @@ -0,0 +1,14 @@ +//! One field of a planner runtime passed as a Python object. + +use crate::planner_phases::_helpers::inputs::runtime; +use crate::planner_phases::models::PlannerRuntime; +use pyo3::prelude::{Bound, Py, PyAny, PyResult}; + +/// One field of a planner runtime passed as a Python object. +#[inline] +pub fn runtime_field<'py>( + runtime: &Bound<'py, PyAny>, + field: fn(&PlannerRuntime) -> &Py, +) -> PyResult> { + runtime::runtime_field(runtime, field) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/warehouse_field.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/warehouse_field.rs new file mode 100644 index 0000000000..35a46a678b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/inputs/warehouse_field.rs @@ -0,0 +1,14 @@ +//! One field of a planner warehouse state passed as a Python object. + +use crate::planner_phases::_helpers::inputs::runtime; +use crate::planner_phases::models::PlannerWarehouseState; +use pyo3::prelude::{Bound, Py, PyAny, PyResult}; + +/// One field of a planner warehouse state passed as a Python object. +#[inline] +pub fn warehouse_field<'py>( + warehouse: &Bound<'py, PyAny>, + field: fn(&PlannerWarehouseState) -> &Py, +) -> PyResult> { + runtime::warehouse_field(warehouse, field) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/mod.rs new file mode 100644 index 0000000000..c73411eada --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/mod.rs @@ -0,0 +1,3 @@ +pub mod inputs; +pub mod outputs; +pub mod rebuilds; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_deferred_origin_notes.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_deferred_origin_notes.rs new file mode 100644 index 0000000000..0bbac7ae9d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_deferred_origin_notes.rs @@ -0,0 +1,72 @@ +//! Note unselected upstreams deferred to pre-migration names, and each lookup in `failures`. + +use crate::planner_phases::_helpers::outputs::output_assembly::concat; +use crate::planner_phases::main::outputs::unselected_upstream_models::unselected_upstream_models; +use crate::planner_phases::models::PlanOutputFields; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; + +/// Note unselected upstreams deferred to pre-migration names, and each lookup in `failures`. +pub fn add_deferred_origin_notes<'py>( + mut output: PlanOutputFields<'py>, + (project, scopes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (deferral, failures): (&Bound<'py, PyAny>, &[(String, String)]), +) -> PyResult> { + let py = project.py(); + let locations = native_field(deferral, "deferred_locations")?; + if locations.is_none() { + return Ok(output); + } + let names: Vec = unselected_upstream_models(scopes)?; + let warning = |name: &str, message: &str| -> PyResult> { + native( + py, + RecordId::PlanWarning, + &[ + ("model_name", text(py, name)), + ( + "severity", + enum_member(py, RecordId::WarningSeverity, "WARNING")?, + ), + ("message", text(py, message)), + ], + ) + }; + let mut notes: Vec> = Vec::new(); + for model in native_field(project, "models")?.try_iter()? { + let model = model?; + let name: String = native_field(&model, "name")?.extract()?; + if names.binary_search(&name).is_err() { + continue; + } + let Some(location) = locations.downcast::()?.get_item(&name)? else { + continue; + }; + if native_field(&location, "name")? + .eq(native_field(&native_field(&model, "destination")?, "name")?)? + { + continue; + } + notes.push(warning( + &name, + &format!( + "deferred to pre-migration name {}: the deferred target has not built model '{name}' under its current name yet", + native_field(&location, "qualified_name")? + ), + )?); + } + for (name, message) in failures { + notes.push(warning(name, message)?); + } + if !notes.is_empty() { + let warnings = concat(py, &[output.field("warnings")?, tuple(py, notes)?])?; + output.set("warnings", warnings); + } + Ok(output) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_metadata.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_metadata.rs new file mode 100644 index 0000000000..c6605b8674 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_metadata.rs @@ -0,0 +1,33 @@ +//! Add the direct-mode source freshness, remaining stale models and selection diagnostics metadata. + +use crate::planner_phases::_helpers::outputs::output_assembly::freshness_metadata; +use crate::planner_phases::models::PlanOutputFields; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_phases::main::sorted_names::sorted_names; +use sqlbuild_python_project::planner_phases::main::text::text; + +/// Add the direct-mode source freshness, remaining stale models and selection diagnostics metadata. +pub fn add_plan_metadata<'py>( + mut output: PlanOutputFields<'py>, + source_freshness: &Bound<'py, PyAny>, + policies: &Bound<'py, PyAny>, +) -> PyResult> { + let py = source_freshness.py(); + let diagnostics = PyDict::new(py); + diagnostics.set_item("mode", text(py, "direct"))?; + diagnostics.set_item("enabled", native_field(policies, "selection_diagnostics")?)?; + output.metadata.push(( + "direct_source_freshness", + freshness_metadata(source_freshness)?, + )); + output.metadata.push(( + "direct_remaining_stale_model_names", + sorted_names(py, Vec::new())?, + )); + output + .metadata + .push(("selection_diagnostics", diagnostics.into_any())); + Ok(output) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_warnings.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_warnings.rs new file mode 100644 index 0000000000..3eb7def0e1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_plan_warnings.rs @@ -0,0 +1,29 @@ +//! Add unknown source freshness warnings and, with selection diagnostics, stale-selection warnings. + +use crate::planner_phases::_helpers::outputs::output_assembly; +use crate::planner_phases::models::{PlanOutputFields, PlanWarningContext}; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Add unknown source freshness warnings and, with selection diagnostics, stale-selection warnings. +#[inline] +pub fn add_plan_warnings<'py>( + output: PlanOutputFields<'py>, + PlanWarningContext { + scopes, + warehouse, + identities, + stale_warning_changes, + }: PlanWarningContext<'_, 'py>, + (source_freshness, policies): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + output_assembly::add_plan_warnings( + output, + PlanWarningContext { + scopes, + warehouse, + identities, + stale_warning_changes, + }, + (source_freshness, policies), + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_storage_policies.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_storage_policies.rs new file mode 100644 index 0000000000..fa3d16dae6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/add_storage_policies.rs @@ -0,0 +1,27 @@ +//! Add migration fingerprints, rename hints, migration, table-type and retention entries. + +use crate::planner_phases::_helpers::outputs::output_assembly; +use crate::planner_phases::models::{PlanOutputFields, StoragePolicyContext}; +use pyo3::prelude::PyResult; + +/// Add migration fingerprints, rename hints, migration, table-type and retention entries. +#[inline] +pub fn add_storage_policies<'py>( + output: PlanOutputFields<'py>, + StoragePolicyContext { + runtime, + warehouse, + scopes, + policies, + }: StoragePolicyContext<'_, 'py>, +) -> PyResult> { + output_assembly::add_storage_policies( + output, + StoragePolicyContext { + runtime, + warehouse, + scopes, + policies, + }, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/base_plan_output.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/base_plan_output.rs new file mode 100644 index 0000000000..bae81b84cf --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/base_plan_output.rs @@ -0,0 +1,39 @@ +//! The plan output's base fields from the planned entries, relations and source freshness. + +use crate::planner_phases::_helpers::outputs::output_assembly; +use crate::planner_phases::models::{PlanOutputContext, PlanOutputFields, PlannedModelEntries}; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// The plan output's base fields from the planned entries, relations and source freshness. +#[inline] +pub fn base_plan_output<'py>( + PlanOutputContext { + runtime, + warehouse, + identities, + overrides, + }: PlanOutputContext<'_, 'py>, + PlannedModelEntries { + execution_scope, + reconciliation, + entries, + source_freshness, + }: PlannedModelEntries<'_, 'py>, + planned_sql_tests: &Bound<'py, PyAny>, +) -> PyResult> { + output_assembly::base_plan_output( + PlanOutputContext { + runtime, + warehouse, + identities, + overrides, + }, + PlannedModelEntries { + execution_scope, + reconciliation, + entries, + source_freshness, + }, + planned_sql_tests, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/build_planner_identity_context.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/build_planner_identity_context.rs new file mode 100644 index 0000000000..a1cdf347d2 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/build_planner_identity_context.rs @@ -0,0 +1,39 @@ +//! Build planner identity context. + +use crate::planner_phases::_helpers::inputs::contexts::class_field; +use crate::planner_phases::_helpers::outputs::identities::version_identities; +use crate::planner_phases::models::PlannerIdentityContext; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; + +/// Build planner identity context. +pub fn build_planner_identity_context<'py>( + project: &Bound<'py, PyAny>, + (scopes, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + include_stale_warning_identities: bool, +) -> PyResult> { + let recorded_fingerprints: Bound<'py, PyAny> = native_field(snapshot, "fingerprints")?; + let identities = version_identities( + project, + class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, + &recorded_fingerprints, + )?; + let stale = if include_stale_warning_identities { + version_identities( + project, + class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)?, + &recorded_fingerprints, + )? + } else { + identities.clone() + }; + Ok(Bound::new( + project.py(), + PlannerIdentityContext { + version_identities: identities.unbind(), + stale_warning_identities: stale.unbind(), + }, + )? + .into_any()) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/check_selected_scope_buildability.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/check_selected_scope_buildability.rs new file mode 100644 index 0000000000..8090e77bf5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/check_selected_scope_buildability.rs @@ -0,0 +1,15 @@ +//! Check selected scope buildability. + +use crate::planner_phases::_helpers::outputs::buildability; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Check selected scope buildability. +#[inline] +pub fn check_selected_scope_buildability<'py>( + runtime: &Bound<'py, PyAny>, + scopes: &Bound<'py, PyAny>, + warehouse: &Bound<'py, PyAny>, + deferral: &Bound<'py, PyAny>, +) -> PyResult<()> { + buildability::check_selected_scope_buildability(runtime, scopes, warehouse, deferral) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes.rs new file mode 100644 index 0000000000..5a1378b311 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes.rs @@ -0,0 +1,18 @@ +//! Changes of stale-warning models whose identities differ from the execution identities. + +use crate::planner_phases::_helpers::outputs::identities; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Changes of stale-warning models whose identities differ from the execution identities. +#[inline] +pub fn detect_stale_warning_changes<'py>( + (project, scopes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + snapshot: &Bound<'py, PyAny>, + (identities, execution_changes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + identities::detect_stale_warning_changes( + (project, scopes), + snapshot, + (identities, execution_changes), + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes_of_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes_of_function.rs new file mode 100644 index 0000000000..be2b04d3fd --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/detect_stale_warning_changes_of_function.rs @@ -0,0 +1,10 @@ +//! `detect_stale_warning_changes_of` as a function of `module`. + +use crate::planner_phases::_helpers::outputs::identities::detect_stale_warning_changes_of; + +/// `detect_stale_warning_changes_of` as a function of `module`. +pub fn detect_stale_warning_changes_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(detect_stale_warning_changes_of, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs new file mode 100644 index 0000000000..69b6f2dbe9 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs @@ -0,0 +1,11 @@ +pub mod add_deferred_origin_notes; +pub mod add_plan_metadata; +pub mod add_plan_warnings; +pub mod add_storage_policies; +pub mod base_plan_output; +pub mod build_planner_identity_context; +pub mod check_selected_scope_buildability; +pub mod detect_stale_warning_changes; +pub mod detect_stale_warning_changes_of_function; +pub mod planner_identity_work_info_function; +pub mod unselected_upstream_models; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/planner_identity_work_info_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/planner_identity_work_info_function.rs new file mode 100644 index 0000000000..51d1b092e1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/planner_identity_work_info_function.rs @@ -0,0 +1,10 @@ +//! `planner_identity_work_info` as a function of `module`. + +use crate::planner_phases::_helpers::outputs::identities::planner_identity_work_info; + +/// `planner_identity_work_info` as a function of `module`. +pub fn planner_identity_work_info_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(planner_identity_work_info, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/unselected_upstream_models.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/unselected_upstream_models.rs new file mode 100644 index 0000000000..0a58ec1c82 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/unselected_upstream_models.rs @@ -0,0 +1,26 @@ +//! The unselected models the selected scope reads, sorted. + +use crate::planner_phases::_helpers::inputs::contexts::class_field; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_analysis::planner_scope::constants::MODEL; +use sqlbuild_python_project::planner_scope::models::PlannerScopeResolution; + +/// The unselected models the selected scope reads, sorted. +pub fn unselected_upstream_models(scopes: &Bound<'_, PyAny>) -> PyResult> { + let selected = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; + let selected = sqlbuild_python_project::planner_scope::main::scope_data::scope_data(&selected)?; + let mut names: Vec = Vec::new(); + for (key, deps) in &selected.upstream { + if !selected.selected.contains(key) { + continue; + } + for dep in deps { + if dep.0 == MODEL && !selected.selected.contains(dep) { + names.push(dep.1.clone()); + } + } + } + names.sort(); + names.dedup(); + Ok(names) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/build_planner_entry_results.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/build_planner_entry_results.rs new file mode 100644 index 0000000000..f13fa09f2f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/build_planner_entry_results.rs @@ -0,0 +1,21 @@ +//! Model plan entries for the pruned execution scope, with the freshness and override inputs they need. + +use crate::planner_phases::_helpers::rebuilds::execution_changes; +use crate::planner_phases::models::{ChangeReconciliation, ModelEntryResults}; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Model plan entries for the pruned execution scope, with the freshness and override inputs they need. +#[inline] +pub fn build_planner_entry_results<'py>( + (runtime, warehouse): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (overrides, deferral): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (execution_scope, reconciliation): (&Bound<'py, PyAny>, &ChangeReconciliation<'py>), + source_freshness: &Bound<'py, PyAny>, +) -> PyResult> { + execution_changes::build_planner_entry_results( + (runtime, warehouse), + (overrides, deferral), + (execution_scope, reconciliation), + source_freshness, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed.rs new file mode 100644 index 0000000000..c032730e9e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed.rs @@ -0,0 +1,41 @@ +//! Check full rebuild allowed. + +use crate::planner_phases::_helpers::rebuilds::full_refresh::configured_full_refresh; +use crate::planner_phases::constants::FULL_REFRESH_DISABLED_REBUILD_CODE; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyBool; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; + +#[pyfunction] +#[pyo3(signature = (*, model, change_result, full_rebuild_cause))] +pub fn check_full_rebuild_allowed( + model: &Bound<'_, PyAny>, + change_result: &Bound<'_, PyAny>, + full_rebuild_cause: &str, +) -> PyResult<()> { + let py = model.py(); + if !configured_full_refresh(model)?.is(PyBool::new(py, false)) { + return Ok(()); + } + if native_field(change_result, "change_kind")?.eq("first_run")? + || native_field(change_result, "query_changed")?.is_truthy()? + { + return Ok(()); + } + let name: String = model_name(model)?; + Err(planner_error( + py, + &format!( + "model '{name}' sets full_refresh false, but the plan would fully rebuild it ({full_rebuild_cause})" + ), + ( + Some(FULL_REFRESH_DISABLED_REBUILD_CODE), + Some( + "set replay_on_change to forward or bounded- on this model to keep its existing history, or remove full_refresh false to allow the rebuild", + ), + ), + )) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed_function.rs new file mode 100644 index 0000000000..f1834b4b6f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/check_full_rebuild_allowed_function.rs @@ -0,0 +1,10 @@ +//! `check_full_rebuild_allowed` as a function of `module`. + +use crate::planner_phases::main::rebuilds::check_full_rebuild_allowed::check_full_rebuild_allowed; + +/// `check_full_rebuild_allowed` as a function of `module`. +pub fn check_full_rebuild_allowed_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(check_full_rebuild_allowed, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/defer_to_migration_origins.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/defer_to_migration_origins.rs new file mode 100644 index 0000000000..b119e3161f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/defer_to_migration_origins.rs @@ -0,0 +1,19 @@ +//! Redirect unselected `models` unbuilt in the deferred target to their `migrate_from` relation. + +use crate::planner_phases::_helpers::rebuilds::deferred_origins; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyDict; + +/// Redirect unselected `models` unbuilt in the deferred target to their `migrate_from` relation. +#[inline] +pub fn defer_to_migration_origins<'py>( + (project, adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + (locations, relations): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), + models: &[String], +) -> PyResult> { + deferred_origins::defer_to_migration_origins( + (project, adapter, connection), + (locations, relations), + models, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause.rs new file mode 100644 index 0000000000..bd8115348a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause.rs @@ -0,0 +1,25 @@ +//! Describe full rebuild cause. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_analysis::planner_warehouse::main::full_rebuild_cause::full_rebuild_cause; +use sqlbuild_analysis::planner_warehouse::models::FullRebuildFacts; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_phases::main::strings::strings; + +#[pyfunction] +#[pyo3(signature = (*, change_result, full_refresh))] +pub fn describe_full_rebuild_cause( + change_result: &Bound<'_, PyAny>, + full_refresh: bool, +) -> PyResult { + Ok(full_rebuild_cause(&FullRebuildFacts { + full_refresh, + change_kind: native_field(change_result, "change_kind")? + .str()? + .extract::()?, + query_changed: native_field(change_result, "query_changed")?.is_truthy()?, + changed_functions: strings(&native_field(change_result, "changed_functions")?)?, + has_schema_findings: native_field(change_result, "schema_findings")?.is_truthy()?, + })) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause_function.rs new file mode 100644 index 0000000000..61c65cebe4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/describe_full_rebuild_cause_function.rs @@ -0,0 +1,10 @@ +//! `describe_full_rebuild_cause` as a function of `module`. + +use crate::planner_phases::main::rebuilds::describe_full_rebuild_cause::describe_full_rebuild_cause; + +/// `describe_full_rebuild_cause` as a function of `module`. +pub fn describe_full_rebuild_cause_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(describe_full_rebuild_cause, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effective_target_config.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effective_target_config.rs new file mode 100644 index 0000000000..359c74b55a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effective_target_config.rs @@ -0,0 +1,20 @@ +//! The resolved config of the planned target, or defaults without one. + +use crate::planner_phases::_helpers::inputs::runtime::runtime_field; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_project::planner_phases::main::native::native; + +/// The resolved config of the planned target, or defaults without one. +pub fn effective_target_config<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { + let py = runtime.py(); + let project_config = runtime_field(runtime, |runtime| &runtime.project_config)?; + let local_config = runtime_field(runtime, |runtime| &runtime.local_config)?; + let target_name = native_field(&native_field(runtime, "project")?, "effective_target_name")?; + if project_config.is_none() || local_config.is_none() || target_name.is_none() { + return native(py, RecordId::TargetConfig, &[]); + } + resolve_target_config(py, &project_config, &local_config, &target_name) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs new file mode 100644 index 0000000000..2e93d90235 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs @@ -0,0 +1,22 @@ +//! Names of models effectively full refreshed for this invocation. + +use crate::planner_phases::main::rebuilds::resolve_model_full_refresh::resolve_model_full_refresh; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; + +/// Names of models effectively full refreshed for this invocation. +pub fn effectively_full_refreshed_model_names<'py>( + project: &Bound<'py, PyAny>, + cli_full_refresh: bool, +) -> PyResult> { + let mut names: Vec<_> = Vec::new(); + for model in compiled_models(project)?.try_iter()? { + let model = model?; + if resolve_model_full_refresh(&model, cli_full_refresh)? { + names.push(name_object(&model)?); + } + } + frozenset(project.py(), &names) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/mod.rs new file mode 100644 index 0000000000..b3c1c34f26 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/mod.rs @@ -0,0 +1,13 @@ +pub mod build_planner_entry_results; +pub mod check_full_rebuild_allowed; +pub mod check_full_rebuild_allowed_function; +pub mod defer_to_migration_origins; +pub mod describe_full_rebuild_cause; +pub mod describe_full_rebuild_cause_function; +pub mod effective_target_config; +pub mod effectively_full_refreshed_model_names; +pub mod plan_retention_function; +pub mod plan_table_types_function; +pub mod reconcile_execution_changes; +pub mod resolve_model_full_refresh; +pub mod table_type_copy_name_function; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_retention_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_retention_function.rs new file mode 100644 index 0000000000..a80cee656b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_retention_function.rs @@ -0,0 +1,10 @@ +//! `plan_retention` as a function of `module`. + +use crate::planner_phases::_helpers::rebuilds::retention::plan_retention; + +/// `plan_retention` as a function of `module`. +pub fn plan_retention_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(plan_retention, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_table_types_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_table_types_function.rs new file mode 100644 index 0000000000..308dc0630f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/plan_table_types_function.rs @@ -0,0 +1,10 @@ +//! `plan_table_types` as a function of `module`. + +use crate::planner_phases::_helpers::rebuilds::retention::plan_table_types; + +/// `plan_table_types` as a function of `module`. +pub fn plan_table_types_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(plan_table_types, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/reconcile_execution_changes.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/reconcile_execution_changes.rs new file mode 100644 index 0000000000..1a5e191553 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/reconcile_execution_changes.rs @@ -0,0 +1,41 @@ +//! The execution scope's changes with honest write hashes, and the actions merged from them. + +use crate::planner_identity::main::merge_recomputed_model_changes::merge_recomputed_model_changes; +use crate::planner_identity::main::with_honest_model_write_hashes::with_honest_model_write_hashes; +use crate::planner_phases::_helpers::inputs::contexts::class_field; +use crate::planner_phases::models::ChangeReconciliation; +use crate::planner_phases::models::PlannerIdentityContext; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::boundary::main::keywords::keywords; + +/// The execution scope's changes with honest write hashes, and the actions merged from them. +pub fn reconcile_execution_changes<'py>( + (execution_scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + identities: &Bound<'py, PyAny>, + resolved_actions: &Bound<'py, PyAny>, + changes: &Bound<'py, PyAny>, +) -> PyResult> { + let py = execution_scope.py(); + let honest = with_honest_model_write_hashes( + py, + Some(keywords( + py, + &[ + ("scope", execution_scope.clone()), + ("snapshot", snapshot.clone()), + ("changes", changes.clone()), + ( + "version_identities", + class_field(identities, |i: &PlannerIdentityContext| { + &i.version_identities + })?, + ), + ], + )?), + )?; + let resolved = merge_recomputed_model_changes(resolved_actions, &honest)?; + Ok(ChangeReconciliation { + changes: honest, + resolved_actions: resolved, + }) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/resolve_model_full_refresh.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/resolve_model_full_refresh.rs new file mode 100644 index 0000000000..0e073d4dff --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/resolve_model_full_refresh.rs @@ -0,0 +1,17 @@ +//! dbt-compatible nullable model full-refresh semantics. + +use crate::planner_phases::_helpers::rebuilds::full_refresh::configured_full_refresh; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyBool; + +/// dbt-compatible nullable model full-refresh semantics. +pub fn resolve_model_full_refresh( + model: &Bound<'_, PyAny>, + cli_full_refresh: bool, +) -> PyResult { + let configured = configured_full_refresh(model)?; + if configured.is_instance_of::() { + return configured.extract(); + } + Ok(cli_full_refresh) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/table_type_copy_name_function.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/table_type_copy_name_function.rs new file mode 100644 index 0000000000..56017b44d1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/table_type_copy_name_function.rs @@ -0,0 +1,10 @@ +//! `table_type_copy_name` as a function of `module`. + +use crate::planner_phases::_helpers::rebuilds::retention::table_type_copy_name; + +/// `table_type_copy_name` as a function of `module`. +pub fn table_type_copy_name_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(table_type_copy_name, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/mod.rs new file mode 100644 index 0000000000..94b56e8614 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/mod.rs @@ -0,0 +1,6 @@ +//! Native execution-planning phases over Python planner models and adapter callbacks. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/models.rs b/crates/sqlbuild-python-planner/src/planner_phases/models.rs new file mode 100644 index 0000000000..a7a3da7fd2 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/models.rs @@ -0,0 +1,106 @@ +//! Public records of planner phases. + +use pyo3::prelude::{Bound, Py, PyAny}; +use pyo3::pyclass; +use std::collections::BTreeMap; + +/// The plan output's fields and metadata entries, each set before the output is built. +#[derive(Debug)] +pub struct PlanOutputFields<'py> { + pub fields: Vec<(&'static str, Bound<'py, PyAny>)>, + pub metadata: Vec<(&'static str, Bound<'py, PyAny>)>, +} + +/// The model changes and resolved actions after write-hash reconciliation. +#[derive(Debug)] +pub struct ChangeReconciliation<'py> { + pub changes: Bound<'py, PyAny>, + pub resolved_actions: Bound<'py, PyAny>, +} + +/// Planned model entries and the warnings planning them raised. +#[derive(Debug)] +pub struct ModelEntryResults<'py> { + pub entries: Vec>, + pub warnings: Vec>, +} + +/// The runtime, warehouse state, identities and overrides the base plan output reads. +#[derive(Debug)] +pub struct PlanOutputContext<'a, 'py> { + pub runtime: &'a Bound<'py, PyAny>, + pub warehouse: &'a Bound<'py, PyAny>, + pub identities: &'a Bound<'py, PyAny>, + pub overrides: &'a Bound<'py, PyAny>, +} + +/// The execution scope, reconciled changes, planned entries and source freshness of a plan. +#[derive(Debug)] +pub struct PlannedModelEntries<'a, 'py> { + pub execution_scope: &'a Bound<'py, PyAny>, + pub(crate) reconciliation: &'a ChangeReconciliation<'py>, + pub entries: ModelEntryResults<'py>, + pub source_freshness: &'a Bound<'py, PyAny>, +} + +/// The runtime, warehouse state, scopes and policies storage policy planning reads. +#[derive(Debug)] +pub struct StoragePolicyContext<'a, 'py> { + pub runtime: &'a Bound<'py, PyAny>, + pub warehouse: &'a Bound<'py, PyAny>, + pub scopes: &'a Bound<'py, PyAny>, + pub policies: &'a Bound<'py, PyAny>, +} + +/// The scopes, warehouse state, identities and changes stale-selection warnings read. +#[derive(Debug)] +pub struct PlanWarningContext<'a, 'py> { + pub scopes: &'a Bound<'py, PyAny>, + pub warehouse: &'a Bound<'py, PyAny>, + pub identities: &'a Bound<'py, PyAny>, + pub stale_warning_changes: &'a Bound<'py, PyAny>, +} + +/// Resolved execution environment shared by planner phases. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct PlannerRuntime { + pub project: Py, + pub adapter: Py, + pub connection: Py, + pub project_config: Py, + pub local_config: Py, + pub on_progress: Py, + pub invocation_time: Py, +} + +/// Warehouse snapshot, inspection relations and migration plans gathered once per plan. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct PlannerWarehouseState { + pub snapshot: Py, + pub inspection_relations: Py, + pub migration_entries: Py, + pub migration_warnings: Py, + pub column_migration_entries: Py, + pub column_rename_hints: Py, + pub old_name_view_entries: Py, + pub old_name_views: Py, + pub migration_fingerprints: Py, +} + +/// Expected version identities for inspection and stale-warning scopes. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub struct PlannerIdentityContext { + pub version_identities: Py, + pub stale_warning_identities: Py, +} + +/// Planner identity work since the last reset, for work-count tests. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct IdentityWork { + pub identity_builds: u64, + pub stale_detections: BTreeMap, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_bounds.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_bounds.rs index 4c686ff2ed..d42cd6fc87 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_bounds.rs @@ -2,11 +2,11 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::planner_resolution::maximum_safety::apply_maximum_start_policy; -use crate::bindings::_helpers::planner_resolution::values::string; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::planner_resolution::_helpers::references::values::string; +use crate::planner_resolution::_helpers::safety::maximum_safety::apply_maximum_start_policy; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyDict; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_cursor::algebra::main::inclusive_to_exclusive::inclusive_to_exclusive; use sqlbuild_cursor::algebra::models::{DurationSpan, Grain, Scalar}; use sqlbuild_cursor::planner::main::raw_bounds::raw_bounds; @@ -24,7 +24,7 @@ use sqlbuild_python_front::cursor::main::conversion::algebra_error::algebra_erro use sqlbuild_python_front::cursor::main::scalars::try_parse::try_parse; use sqlbuild_python_front::cursor::models::CursorSession; -pub(super) fn optional_scalar<'py>( +pub(crate) fn optional_scalar<'py>( session: &CursorSession<'py>, value: &Bound<'py, PyAny>, ) -> PyResult> { @@ -35,7 +35,7 @@ pub(super) fn optional_scalar<'py>( } } -pub(super) fn scalars<'py>( +pub(crate) fn scalars<'py>( session: &CursorSession<'py>, object: &Bound<'py, PyAny>, field: &str, @@ -71,7 +71,7 @@ fn read_snapshot<'py>( }) } -pub(super) fn duration(value: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn duration(value: &Bound<'_, PyAny>) -> PyResult> { if value.is_none() { return Ok(None); } @@ -159,7 +159,7 @@ pub(crate) fn apply_typed_cursor_replay_policy<'py>( session.python_scalar(&result) } -pub(super) fn required_scalar<'py>( +pub(crate) fn required_scalar<'py>( session: &CursorSession<'py>, raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, @@ -299,8 +299,3 @@ pub(crate) fn compute_cursor_bounds<'py>( } apply_maximum_start_policy(py, Some(keywords(py, &policy_args)?)) } - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(compute_cursor_bounds, module)?)?; - module.add_function(wrap_pyfunction!(apply_typed_cursor_replay_policy, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs index 5cbd26a115..f518f20798 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs @@ -2,22 +2,22 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; -use crate::bindings::_helpers::plan_output::display::resolve_bounded_cursor_override; -use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_resolution::constants::CURSOR_START; -use crate::bindings::_helpers::planner_resolution::cursor_bounds::compute_cursor_bounds; -use crate::bindings::_helpers::planner_resolution::cursor_policies::{ +use crate::plan_output::main::resolve_bounded_cursor_override::resolve_bounded_cursor_override; +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_phases::main::inputs::class_field::class_field; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::compute_cursor_bounds; +use crate::planner_resolution::_helpers::cursor::cursor_policies::{ future_cursor_config, start_cursor_config, }; -use crate::bindings::_helpers::planner_resolution::cursor_windows::cursor_inputs_without_rows; -use crate::bindings::_helpers::planner_resolution::cursor_windows::empty_cursor_window; -use crate::bindings::_helpers::planner_resolution::cursor_windows::empty_input_rebuild_refusal; -use crate::bindings::_helpers::planner_resolution::cursor_windows::unreadable_cursor_inputs_error; -use crate::bindings::_helpers::planner_resolution::cursor_windows::without_destination_cursor; -use crate::bindings::_helpers::planner_resolution::future_safety::resolve_future_cursor_safety; -use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest; -use crate::bindings::_helpers::planner_resolution::values::string; +use crate::planner_resolution::_helpers::cursor::cursor_windows::cursor_inputs_without_rows; +use crate::planner_resolution::_helpers::cursor::cursor_windows::empty_cursor_window; +use crate::planner_resolution::_helpers::cursor::cursor_windows::empty_input_rebuild_refusal; +use crate::planner_resolution::_helpers::cursor::cursor_windows::unreadable_cursor_inputs_error; +use crate::planner_resolution::_helpers::cursor::cursor_windows::without_destination_cursor; +use crate::planner_resolution::_helpers::references::values::string; +use crate::planner_resolution::_helpers::safety::future_safety::resolve_future_cursor_safety; +use crate::planner_resolution::constants::CURSOR_START; +use crate::planner_resolution::models::ModelSqlRequest; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::{IntoPyObject, PyErr}; use sqlbuild_python_base::boundary::main::call_with::call_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs index b5d65ccb4c..a6b2ad573a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs @@ -1,11 +1,9 @@ //! Selected-model inspection and untyped override conversion. -use crate::bindings::_helpers::planner_resolution::values::{method, string}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use crate::planner_resolution::_helpers::references::values::{method, string}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cursor::planner::main::override_kind::override_kind; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -28,7 +26,7 @@ const TIMESTAMP_CURSOR: &str = "timestamp"; #[pyfunction(name = "type_cursor_overrides")] #[pyo3(signature = (**kwargs))] -fn type_cursor_overrides_of<'py>( +pub(crate) fn type_cursor_overrides_of<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -95,7 +93,3 @@ pub(crate) fn type_cursor_overrides<'py>( let kind = enum_from_value(py, RecordId::CursorType, kind)?; method(overrides, "typed_as", &[("cursor_type", kind)]) } - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(type_cursor_overrides_of, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_policies.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_policies.rs index 50a8bcd9a5..978a01dd64 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_policies.rs @@ -82,11 +82,3 @@ pub(crate) fn start_cursor_config<'py>( ) -> PyResult> { resolve_policy(model, project_config, &START_POLICY) } - -/// Resolve automatic-start policy with model config taking precedence. -pub(crate) fn resolve_start_cursor_config<'py>( - model: &Bound<'py, PyAny>, - project_config: &Bound<'py, PyAny>, -) -> PyResult> { - start_cursor_config(model, project_config) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs similarity index 64% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs index 605b995324..31a7c01bb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs @@ -1,27 +1,25 @@ //! Plan-entry cursor windows: overrides, planner-owned bounds and microbatch ranges. +use crate::planner_resolution::main::cursor::config_str::config_str; +use crate::planner_resolution::main::cursor::model_cursor_overrides::model_cursor_overrides; +use crate::planner_resolution::main::cursor::plan_cursor_bounds::plan_cursor_bounds; +use crate::planner_resolution::main::cursor::resolve_effective_timestamp_grain::resolve_effective_timestamp_grain; +use crate::planner_resolution::models::CursorWindowInputs; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; -use crate::bindings::_helpers::plan_output::display::resolve_bounded_cursor_override; -use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_resolution::cursor_bounds::compute_cursor_bounds; -use crate::bindings::_helpers::planner_resolution::cursor_bounds::duration; -use crate::bindings::_helpers::planner_resolution::cursor_policies::{ - future_cursor_config, start_cursor_config, -}; -use crate::bindings::_helpers::planner_resolution::cursor_safety::{replace, utc_clock}; -use crate::bindings::_helpers::planner_resolution::cursor_windows::cursor_inputs_without_rows; -use crate::bindings::_helpers::planner_resolution::cursor_windows::normalize_cursor_snapshot_grain; -use crate::bindings::_helpers::planner_resolution::cursor_windows::resolve_effective_timestamp_grain; -use crate::bindings::_helpers::planner_resolution::cursor_windows::without_destination_cursor; -use crate::bindings::_helpers::planner_resolution::future_safety::resolve_future_cursor_safety; -use crate::bindings::_helpers::planner_resolution::values::string; -use crate::bindings::_helpers::planner_strategy::model_entries::resolve_microbatch_lookback; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::plan_output::main::resolve_bounded_cursor_override::resolve_bounded_cursor_override; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::compute_cursor_bounds; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::duration; +use crate::planner_resolution::_helpers::cursor::cursor_safety::{replace, utc_clock}; +use crate::planner_resolution::_helpers::cursor::cursor_windows::normalize_cursor_snapshot_grain; +use crate::planner_resolution::_helpers::cursor::cursor_windows::without_destination_cursor; +use crate::planner_resolution::_helpers::references::values::string; +use crate::planner_resolution::_helpers::safety::future_safety::resolve_future_cursor_safety; +use crate::planner_strategy::main::model_entries::resolve_microbatch_lookback::resolve_microbatch_lookback; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, PyErr, pyfunction}; use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::models::{Scalar, ScalarValue, Timestamp, ZoneOffset}; use sqlbuild_cursor::planner::main::cursor_end_clamp::cursor_end_clamp; @@ -42,19 +40,18 @@ use sqlbuild_python_front::cursor::main::values::local_date_time::local_date_tim use sqlbuild_python_front::cursor::main::values::replaced::replaced; use sqlbuild_python_front::cursor::models::CursorSession; use sqlbuild_python_front::model_config::main::config::config_cursor_bound::config_cursor_bound as typed_cursor_bound; -use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::planner_phases::main::object::object; use std::cmp::Ordering; -const INCREMENTAL: &str = "incremental"; -const MICROBATCH: &str = "microbatch"; -const ROLLING_WINDOW: &str = "rolling_window"; -const TIMESTAMP: &str = "timestamp"; -const INTEGER: &str = "integer"; +pub(crate) const INCREMENTAL: &str = "incremental"; +pub(crate) const MICROBATCH: &str = "microbatch"; +pub(crate) const ROLLING_WINDOW: &str = "rolling_window"; +pub(crate) const TIMESTAMP: &str = "timestamp"; +pub(crate) const INTEGER: &str = "integer"; const BOUNDED_BACKFILL: &str = "bounded"; -const FULL_BACKFILL: &str = "full"; +pub(crate) const FULL_BACKFILL: &str = "full"; const WINDOW_REQUIRED: [&str; 7] = [ "model", "snapshot", @@ -72,26 +69,6 @@ const WINDOW_OPTIONAL: [&str; 5] = [ "cursor_input_relations", ]; -/// The planner inputs that decide one model's plan-time cursor window. -pub(crate) struct CursorWindowInputs<'py> { - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) backfill: Bound<'py, PyAny>, - pub(crate) overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), - pub(crate) runtime_owned: bool, - pub(crate) policies: (Bound<'py, PyAny>, Bound<'py, PyAny>), - pub(crate) invocation_time: Bound<'py, PyAny>, - pub(crate) full_refresh: bool, - pub(crate) replaces_relation: bool, - pub(crate) cursor_input_relations: Bound<'py, PyAny>, -} - -/// One string-valued model configuration entry. -pub(crate) fn config_str<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { - let py: Python<'py> = model.py(); - Ok(config_text_object(py, &model_values(model)?, key)? - .unwrap_or_else(|| py.None().into_bound(py))) -} - /// One cursor-bound model configuration entry as its token text. pub(crate) fn config_bound<'py>( model: &Bound<'py, PyAny>, @@ -120,7 +97,7 @@ fn order_scalar<'py>( session.scalar(&parse_value(token, &string(py, kind))?) } -fn order<'py>( +pub(crate) fn order<'py>( session: &CursorSession<'py>, tokens: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), cursor_type: &Bound<'py, PyAny>, @@ -130,7 +107,10 @@ fn order<'py>( session.checked(compare(&left, &right, session)) } -fn token<'py>(py: Python<'py>, bound: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn token<'py>( + py: Python<'py>, + bound: &Bound<'py, PyAny>, +) -> PyResult> { Ok(PyString::new(py, &sentinel_to_token(bound)?).into_any()) } @@ -139,7 +119,7 @@ fn sentinel<'py>(py: Python<'py>, name: &str) -> PyResult> { } /// `bounds` clamped so a concrete window never reaches the exclusive configured `cursor_end`. -fn clamp_cursor_end<'py>( +pub(crate) fn clamp_cursor_end<'py>( bounds: Bound<'py, PyAny>, model: &Bound<'py, PyAny>, ) -> PyResult> { @@ -282,7 +262,7 @@ fn apply_future_safety<'py>( ) } -fn bounded_override<'py>( +pub(crate) fn bounded_override<'py>( model: &Bound<'py, PyAny>, inputs: &CursorWindowInputs<'py>, ) -> PyResult> { @@ -305,14 +285,14 @@ fn backfill_duration<'py>(backfill: &Bound<'py, PyAny>) -> PyResult { - cursor_snapshot: Bound<'py, PyAny>, - typing: (Bound<'py, PyAny>, Bound<'py, PyAny>), - lookback: Bound<'py, PyAny>, - is_microbatch: bool, +pub(crate) struct BoundsRequest<'py> { + pub(crate) cursor_snapshot: Bound<'py, PyAny>, + pub(crate) typing: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) lookback: Bound<'py, PyAny>, + pub(crate) is_microbatch: bool, } -fn snapshot_bounds<'py>( +pub(crate) fn snapshot_bounds<'py>( model: &Bound<'py, PyAny>, inputs: &CursorWindowInputs<'py>, request: BoundsRequest<'py>, @@ -358,55 +338,17 @@ fn snapshot_bounds<'py>( clamp_cursor_end(bounds, model) } -fn model_cursor_snapshot<'py>( +pub(crate) fn model_cursor_snapshot<'py>( model: &Bound<'py, PyAny>, snapshot: &Bound<'py, PyAny>, ) -> PyResult> { native_field(snapshot, "cursor_snapshots")?.call_method1("get", (name_object(model)?,)) } -fn without_destination<'py>(cursor_snapshot: Bound<'py, PyAny>) -> PyResult> { - without_destination_cursor(&cursor_snapshot) -} - -/// Compute cursor bounds for inclusion on the plan entry. -pub(crate) fn plan_cursor_bounds<'py>( - model: &Bound<'py, PyAny>, - inputs: &CursorWindowInputs<'py>, +pub(crate) fn without_destination<'py>( + cursor_snapshot: Bound<'py, PyAny>, ) -> PyResult> { - let py = model.py(); - if !config_str(model, "materialized")?.eq(INCREMENTAL)? - || config_str(model, "cursor")?.is_none() - { - return Ok(py.None().into_bound(py)); - } - let bounded = bounded_override(model, inputs)?; - if !bounded.is_none() { - return clamp_cursor_end(bounded, model); - } - if inputs.runtime_owned || inputs.full_refresh { - return Ok(py.None().into_bound(py)); - } - let mut cursor_snapshot = model_cursor_snapshot(model, &inputs.snapshot)?; - if cursor_snapshot.is_none() { - return Ok(cursor_snapshot); - } - if native_field(&inputs.backfill, "action")?.eq(FULL_BACKFILL)? { - cursor_snapshot = without_destination(cursor_snapshot)?; - } - snapshot_bounds( - model, - inputs, - BoundsRequest { - cursor_snapshot, - typing: ( - config_str(model, "cursor_type")?, - config_str(model, "cursor_grain")?, - ), - lookback: config_str(model, "lookback")?, - is_microbatch: config_str(model, "incremental_mode")?.eq(MICROBATCH)?, - }, - ) + without_destination_cursor(&cursor_snapshot) } /// Compute the real overall cursor range for microbatch batch splitting. @@ -477,119 +419,10 @@ pub(crate) fn microbatch_range<'py>( ) } -/// Empty inputs that leave a planner-owned window empty, and whether others have rows. -pub(crate) fn empty_cursor_inputs<'py>( - model: &Bound<'py, PyAny>, - inputs: &CursorWindowInputs<'py>, -) -> PyResult<(Bound<'py, PyAny>, bool)> { - let py = model.py(); - let nothing = - || -> PyResult<(Bound<'py, PyAny>, bool)> { Ok((PyTuple::empty(py).into_any(), false)) }; - if inputs.full_refresh - || inputs.runtime_owned - || !config_str(model, "materialized")?.eq(INCREMENTAL)? - || config_str(model, "cursor")?.is_none() - || config_str(model, "microbatch_strategy")?.eq(ROLLING_WINDOW)? - || (!inputs.overrides.0.is_none() && !inputs.overrides.1.is_none()) - { - return nothing(); - } - let cursor_snapshot = model_cursor_snapshot(model, &inputs.snapshot)?; - if cursor_snapshot.is_none() { - return nothing(); - } - let decision = cursor_inputs_without_rows(&cursor_snapshot)?; - Ok(( - native_field(&decision, "input_names")?, - native_field(&decision, "waiting_on_empty_inputs")?.is_truthy()?, - )) -} - -/// The future-end and automatic-start policies effective for one model. -pub(crate) fn cursor_safety_configs<'py>( - model: &Bound<'py, PyAny>, - context: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - Ok(( - future_cursor_config( - model, - &class_field(context, |c: &ModelPlanContext| &c.future_cursor_config)?, - )?, - start_cursor_config( - model, - &class_field(context, |c: &ModelPlanContext| &c.start_cursor_config)?, - )?, - )) -} - -fn compile_input_error(py: Python<'_>, message: String) -> PyErr { +pub(crate) fn compile_input_error(py: Python<'_>, message: String) -> PyErr { new_plain_error(py, ErrorId::CompileInput, &message) } -/// Resolve typed cursor overrides to generic start/end for one model. -pub(crate) fn model_cursor_overrides<'py>( - model: &Bound<'py, PyAny>, - cursor_overrides: &Bound<'py, PyAny>, - (start, end): (Bound<'py, PyAny>, Bound<'py, PyAny>), -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - let py = model.py(); - let cursor_type = config_str(model, "cursor_type")?; - let (mut resolved_start, mut resolved_end) = (start, end); - let typed_fields = if cursor_overrides.is_none() { - None - } else if cursor_type.eq(TIMESTAMP)? { - Some(("start_ts", "end_ts")) - } else if cursor_type.eq(INTEGER)? { - Some(("start_int", "end_int")) - } else { - None - }; - if let Some((start_field, end_field)) = typed_fields { - let typed_start = native_field(cursor_overrides, start_field)?; - if !typed_start.is_none() { - resolved_start = token(py, &typed_start)?; - } - let typed_end = native_field(cursor_overrides, end_field)?; - if !typed_end.is_none() { - resolved_end = token(py, &typed_end)?; - } - } - let session = CursorSession::new(py)?; - let name = name_object(model)?.str()?; - let cursor_start = config_bound(model, "cursor_start")?; - let cursor_end = config_bound(model, "cursor_end")?; - if !resolved_start.is_none() - && !cursor_start.is_none() - && order(&session, (&resolved_start, &cursor_start), &cursor_type)? == Ordering::Less - { - return Err(compile_input_error( - py, - format!( - "model '{name}': explicit start {} is before cursor_start {}", - resolved_start.str()?, - cursor_start.str()? - ), - )); - } - if !cursor_end.is_none() { - for (value, label) in [(&resolved_start, "start"), (&resolved_end, "end")] { - if !value.is_none() - && order(&session, (value, &cursor_end), &cursor_type)? != Ordering::Less - { - return Err(compile_input_error( - py, - format!( - "model '{name}': explicit {label} {} is at or beyond exclusive cursor_end {}", - value.str()?, - cursor_end.str()? - ), - )); - } - } - } - Ok((resolved_start, resolved_end)) -} - fn window_inputs<'py>(args: &KeywordArguments<'py>) -> PyResult> { Ok(CursorWindowInputs { snapshot: args.value("snapshot")?, @@ -617,7 +450,7 @@ fn window_inputs<'py>(args: &KeywordArguments<'py>) -> PyResult( +pub(crate) fn compute_plan_cursor_bounds<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -633,7 +466,7 @@ fn compute_plan_cursor_bounds<'py>( /// Compute the real overall cursor range for microbatch batch splitting. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn compute_microbatch_range<'py>( +pub(crate) fn compute_microbatch_range<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -649,7 +482,7 @@ fn compute_microbatch_range<'py>( /// Resolve typed cursor overrides to generic start/end for one model. #[pyfunction] #[pyo3(signature = (*, model, cursor_overrides, start_cursor_override, end_cursor_override))] -fn resolve_cursor_overrides<'py>( +pub(crate) fn resolve_cursor_overrides<'py>( model: &Bound<'py, PyAny>, cursor_overrides: &Bound<'py, PyAny>, start_cursor_override: Bound<'py, PyAny>, @@ -661,9 +494,3 @@ fn resolve_cursor_overrides<'py>( (start_cursor_override, end_cursor_override), ) } - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(compute_plan_cursor_bounds, module)?)?; - module.add_function(wrap_pyfunction!(compute_microbatch_range, module)?)?; - module.add_function(wrap_pyfunction!(resolve_cursor_overrides, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_safety.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_safety.rs index 052858c4ef..61a5fa8cca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_safety.rs @@ -1,6 +1,6 @@ //! Shared safety horizon conversion and Python evidence construction. -use crate::bindings::_helpers::planner_resolution::cursor_bounds::duration; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::duration; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use sqlbuild_cursor::algebra::models::{DurationSpan, Grain, Scalar}; use sqlbuild_cursor::planner::main::safety_horizon::safety_horizon; @@ -16,11 +16,11 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cursor::models::CursorSession; -pub(super) fn safety_error(py: Python<'_>, class: ErrorId, message: &str) -> PyErr { +pub(crate) fn safety_error(py: Python<'_>, class: ErrorId, message: &str) -> PyErr { new_plain_error(py, class, message) } -pub(super) fn safety_duration(value: &Bound<'_, PyAny>, future: bool) -> PyResult { +pub(crate) fn safety_duration(value: &Bound<'_, PyAny>, future: bool) -> PyResult { if let Some(parsed) = duration(value)? { return Ok(parsed); } @@ -45,17 +45,17 @@ pub(super) fn safety_duration(value: &Bound<'_, PyAny>, future: bool) -> PyResul )) } -pub(super) fn timestamp_bound(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn timestamp_bound(value: &Bound<'_, PyAny>) -> PyResult { let py = value.py(); Ok(value.is_instance(&record_class(py, RecordId::DateValue)?)? || value.is_instance(&record_class(py, RecordId::TimestampValue)?)?) } -pub(super) fn utc_clock<'py>(clock: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn utc_clock<'py>(clock: &Bound<'py, PyAny>) -> PyResult> { clock.call_method1("astimezone", (imported(clock.py(), "datetime", "UTC")?,)) } -pub(super) fn horizon<'py>( +pub(crate) fn horizon<'py>( session: &CursorSession<'py>, discovered: &Bound<'py, PyAny>, clock_and_grain: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -85,7 +85,7 @@ pub(super) fn horizon<'py>( )) } -pub(super) fn replace<'py>( +pub(crate) fn replace<'py>( object: &Bound<'py, PyAny>, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_warnings.rs similarity index 54% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_warnings.rs index 599890d6d9..6d1b17c27e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_warnings.rs @@ -1,17 +1,13 @@ //! Prominent warnings for applied future-end and automatic-start cursor safety. -use crate::bindings::_helpers::planner_resolution::values::string; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyString; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; -const FUTURE_CAPPED_CODE: &str = "FUTURE_CURSOR_CAPPED"; -const START_CAPPED_CODE: &str = "AUTOMATIC_START_CAPPED"; +pub(crate) const FUTURE_CAPPED_CODE: &str = "FUTURE_CURSOR_CAPPED"; +pub(crate) const START_CAPPED_CODE: &str = "AUTOMATIC_START_CAPPED"; fn rendered(evidence: &Bound<'_, PyAny>, field: &str) -> PyResult { render_value(&native_field(evidence, field)?) @@ -65,53 +61,16 @@ pub(crate) fn maximum_start_message(bounds: &Bound<'_, PyAny>) -> PyResult( - warnings: &Bound<'py, PyAny>, - model_name: &Bound<'py, PyAny>, - bounds: &Bound<'py, PyAny>, -) -> PyResult> { - let py = warnings.py(); - let mut all: Vec> = warnings - .try_iter()? - .collect::>>>()?; - for (message, code) in [ - (future_cap_message(bounds)?, FUTURE_CAPPED_CODE), - (maximum_start_message(bounds)?, START_CAPPED_CODE), - ] { - let Some(message) = message else { continue }; - all.push(new_record_with( - py, - RecordId::PlanWarning, - &[ - ("model_name", model_name.clone()), - ( - "severity", - enum_member(py, RecordId::WarningSeverity, "WARNING")?, - ), - ("code", string(py, code)), - ("message", string(py, &message)), - ], - )?); - } - Ok(PyTuple::new(py, all)?.into_any()) -} - /// Render a prominent warning for applied future cursor policy. #[pyfunction] #[pyo3(signature = (bounds, /))] -fn future_cursor_cap_warning(bounds: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn future_cursor_cap_warning(bounds: &Bound<'_, PyAny>) -> PyResult> { future_cap_message(bounds) } /// Render a warning that distinguishes start recovery from future-end capping. #[pyfunction] #[pyo3(signature = (bounds, /))] -fn maximum_start_cap_warning(bounds: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn maximum_start_cap_warning(bounds: &Bound<'_, PyAny>) -> PyResult> { maximum_start_message(bounds) } - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(future_cursor_cap_warning, module)?)?; - module.add_function(wrap_pyfunction!(maximum_start_cap_warning, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_windows.rs similarity index 79% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_windows.rs index 46dcb20ea1..779153ef5d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_windows.rs @@ -1,9 +1,9 @@ //! Empty-window decisions and snapshot grain conversion. -use crate::bindings::_helpers::planner_resolution::values::string; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::planner_resolution::_helpers::references::values::string; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; use sqlbuild_cursor::planner::main::empty_inputs::empty_inputs; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -19,7 +19,7 @@ use sqlbuild_python_front::cursor::models::CursorSession; #[pyfunction] #[pyo3(signature = (*, empty_input_names, waits_for_every_input, inputs_with_rows, has_window_start))] -fn decide_empty_cursor_inputs<'py>( +pub(crate) fn decide_empty_cursor_inputs<'py>( empty_input_names: &Bound<'py, PyAny>, waits_for_every_input: bool, inputs_with_rows: bool, @@ -105,7 +105,7 @@ pub(crate) fn empty_cursor_window<'py>( #[pyfunction] #[pyo3(signature = (*, cursor_type))] -fn cursor_window_override_flags(cursor_type: Option<&str>) -> String { +pub(crate) fn cursor_window_override_flags(cursor_type: Option<&str>) -> String { let suffix = if cursor_type == Some("integer") { "int" } else { @@ -178,41 +178,6 @@ pub(crate) fn without_destination_cursor<'py>( replaced_keywords(cursor_snapshot, &kwargs) } -#[pyfunction] -#[pyo3(signature = (*, cursor_type, downstream_grain, cursor_input_grains, microbatch_strategy=None))] -pub(crate) fn resolve_effective_timestamp_grain<'py>( - cursor_type: &Bound<'py, PyAny>, - downstream_grain: &Bound<'py, PyAny>, - cursor_input_grains: &Bound<'py, PyAny>, - microbatch_strategy: Option<&str>, -) -> PyResult> { - let py = cursor_type.py(); - if !cursor_type.eq("timestamp")? { - return Ok(py.None().into_bound(py)); - } - let mut effective = if downstream_grain.is_truthy()? { - downstream_grain.clone() - } else { - string(py, "second") - }; - if microbatch_strategy == Some("watermark") { - return Ok(effective); - } - let session = CursorSession::new(py)?; - for value in cursor_input_grains.try_iter()? { - let value = value?; - let candidate = if value.is_truthy()? { - value - } else { - string(py, "second") - }; - if session.grain(&candidate)? > session.grain(&effective)? { - effective = candidate; - } - } - Ok(effective) -} - #[pyfunction] #[pyo3(signature = (*, cursor_snapshot, cursor_type, effective_grain))] pub(crate) fn normalize_cursor_snapshot_grain<'py>( @@ -261,12 +226,3 @@ pub(crate) fn normalize_cursor_snapshot_grain<'py>( kwargs.set_item("upstream_end_inputs", PyTuple::new(py, ends)?)?; replaced_keywords(cursor_snapshot, &kwargs) } - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(decide_empty_cursor_inputs, module)?)?; - module.add_function(wrap_pyfunction!(empty_cursor_window, module)?)?; - module.add_function(wrap_pyfunction!(cursor_window_override_flags, module)?)?; - module.add_function(wrap_pyfunction!(empty_input_rebuild_refusal, module)?)?; - module.add_function(wrap_pyfunction!(resolve_effective_timestamp_grain, module)?)?; - module.add_function(wrap_pyfunction!(normalize_cursor_snapshot_grain, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/mod.rs new file mode 100644 index 0000000000..6f046db60c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod cursor_bounds; +pub(crate) mod cursor_model; +pub(crate) mod cursor_overrides; +pub(crate) mod cursor_policies; +pub(crate) mod cursor_ranges; +pub(crate) mod cursor_safety; +pub(crate) mod cursor_warnings; +pub(crate) mod cursor_windows; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/mod.rs new file mode 100644 index 0000000000..22f80fcf37 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod cursor; +pub(crate) mod references; +pub(crate) mod safety; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs new file mode 100644 index 0000000000..5ad8ffd24d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs @@ -0,0 +1,15 @@ +//! Location maps retaining the original compiled Python destination objects. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_project::resource_views::main::resource_reads::destination::destination; +use sqlbuild_python_project::resource_views::main::resource_reads::name::name; + +pub(crate) fn locations<'py>(items: &Bound<'py, PyAny>) -> PyResult> { + let result = PyDict::new(items.py()); + for item in items.try_iter()? { + let item = item?; + result.set_item(name(&item)?, destination(&item)?)?; + } + Ok(result) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/mod.rs new file mode 100644 index 0000000000..6f4a003b08 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod locations; +pub(crate) mod orchestration; +pub(crate) mod refs; +pub(crate) mod sources; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs similarity index 50% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs index 453f203e19..3460b890f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs @@ -1,152 +1,36 @@ //! Model, function and SQL-hook orchestration through adapter-owned rendering. -use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; -use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_resolution::cursor_model::{ +use crate::planner_resolution::_helpers::cursor::cursor_model::{ compute_model_cursor_bounds, unbounded, }; -use crate::bindings::_helpers::planner_resolution::models::{CursorFiltering, ModelSqlRequest}; -use crate::bindings::_helpers::planner_resolution::refs::{ - resolve_dbt_ref_references, resolve_refs, resolve_table_function_references, - resolve_udf_references, -}; -use crate::bindings::_helpers::planner_resolution::sources::resolve_sources; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; -use pyo3::{PyErr, pyfunction, wrap_pyfunction}; +use crate::planner_resolution::main::references::model_hook_entries::model_hook_entries; +use crate::planner_resolution::models::Resolution; +use crate::planner_resolution::models::{CursorFiltering, ModelSqlRequest}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{PyErr, pyfunction}; use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; -use sqlbuild_python_front::attachments::main::assert_no_unresolved_sql_markers::assert_no_unresolved_sql_markers; use sqlbuild_python_front::attachments::main::resolve_cursor_intrinsics::resolve_cursor_intrinsics; use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; use sqlbuild_python_front::model_config::main::config::config_text_is::config_text_is; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; -use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; -use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; const APPEND_STRATEGY: &str = "append"; -/// What resolving one model's, hook's or function's references reads. -pub(crate) struct Resolution<'py> { - pub(crate) adapter: Bound<'py, PyAny>, - external: Bound<'py, PyAny>, - source_map: Bound<'py, PyAny>, - source_warehouse_columns: Bound<'py, PyAny>, - model_locations: Bound<'py, PyAny>, - seed_locations: Bound<'py, PyAny>, - function_locations: Bound<'py, PyAny>, -} - -impl<'py> Resolution<'py> { - /// Resolution against a `ModelPlanContext`'s relations. - pub(crate) fn of_context( - adapter: Bound<'py, PyAny>, - context: &Bound<'py, PyAny>, - external: Bound<'py, PyAny>, - ) -> PyResult { - Ok(Self { - adapter, - external, - source_map: class_field(context, |c: &ModelPlanContext| &c.source_map)?, - source_warehouse_columns: class_field(context, |c: &ModelPlanContext| { - &c.source_warehouse_columns - })?, - model_locations: class_field(context, |c: &ModelPlanContext| &c.model_locations)?, - seed_locations: class_field(context, |c: &ModelPlanContext| &c.seed_locations)?, - function_locations: class_field(context, |c: &ModelPlanContext| &c.function_locations)?, - }) - } - - fn resolve(&self, sql: &str, filtering: &CursorFiltering<'py>) -> PyResult { - let py = self.adapter.py(); - let resolved: String = resolve_sources( - sql, - (&self.source_map, &self.source_warehouse_columns), - &self.adapter, - filtering, - )?; - let resolved: String = resolve_refs( - &resolved, - (&self.model_locations, &self.seed_locations), - &self.adapter, - filtering, - )?; - let resolved: String = resolve_dbt_ref_references(&resolved, Some(&self.external))?; - let locations = if self.function_locations.is_truthy()? { - self.function_locations.clone() - } else { - PyDict::new(py).into_any() - }; - let resolved: String = resolve_udf_references(&resolved, &locations, &self.adapter)?; - resolve_table_function_references(&resolved, &locations, &self.adapter) - } - - fn assert_resolved(&self, sql: &str, label: &str) -> PyResult<()> { - let py = self.adapter.py(); - assert_no_unresolved_sql_markers( - py, - &PyString::new(py, sql), - &format!("{label} planned SQL"), - ) - } - - fn unbounded(&self, sql: Bound<'py, PyAny>, label: &str) -> PyResult { - let py = self.adapter.py(); - let resolved = self.resolve( - &sql.extract::()?, - &CursorFiltering { - bounds: py.None().into_bound(py), - filter_inputs: PyDict::new(py).into_any(), - cursor_type: py.None().into_bound(py), - lower_bound_inclusive: true, - }, - )?; - self.assert_resolved(&resolved, label)?; - Ok(resolved) - } - - fn hook(&self, value: Bound<'py, PyAny>, label: &str) -> PyResult> { - let py = value.py(); - if value.is_instance(&record_class(py, RecordId::SqlHookEntry)?)? { - let kwargs = PyDict::new(py); - kwargs.set_item( - "statement", - self.unbounded(native_field(&value, "statement")?, label)?, - )?; - return replaced_keywords(&value, &kwargs); - } - let is_list = value.is_instance_of::(); - if is_list || value.is_instance_of::() { - let mut resolved: Vec> = Vec::new(); - for (index, item) in value.try_iter()?.enumerate() { - resolved.push(self.hook(item?, &format!("{label}[{index}]"))?); - } - return if is_list { - Ok(PyList::new(py, resolved)?.into_any()) - } else { - Ok(PyTuple::new(py, resolved)?.into_any()) - }; - } - Ok(value) - } -} - #[pyfunction(name = "resolve_model_sql")] #[pyo3(signature = (**kwargs))] -fn resolve_model_sql_of<'py>( +pub(crate) fn resolve_model_sql_of<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -295,50 +179,9 @@ fn unresolved_bounds_error(py: Python<'_>, name: &str, cause: Option<&str>) -> P Ok(PyErr::from_value(error.call((message,), Some(&kwargs))?)) } -#[pyfunction] -#[pyo3(signature = (**kwargs))] -pub(crate) fn resolve_function_sql<'py>( - py: Python<'py>, - kwargs: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "resolve_function_sql", - kwargs, - ( - &[ - "adapter", - "function", - "model_locations", - "seed_locations", - "function_locations", - "source_map", - "source_warehouse_columns", - "star_exclude_keyword", - ], - &[], - ), - )?; - let resolution = Resolution { - adapter: args.value("adapter")?, - external: py.None().into_bound(py), - source_map: args.value("source_map")?, - source_warehouse_columns: args.value("source_warehouse_columns")?, - model_locations: args.value("model_locations")?, - seed_locations: args.value("seed_locations")?, - function_locations: args.value("function_locations")?, - }; - let function = args.value("function")?; - let name: String = - sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)? - .extract()?; - let sql: String = resolution.unbounded(body_sql(&function)?, &format!("Function '{name}'"))?; - Ok(PyString::new(py, &sql).into_any()) -} - #[pyfunction] #[pyo3(signature = (*, model, adapter, context, external_sql_reference_resolver))] -fn resolve_model_hook_entries<'py>( +pub(crate) fn resolve_model_hook_entries<'py>( model: &Bound<'py, PyAny>, adapter: Bound<'py, PyAny>, context: Bound<'py, PyAny>, @@ -347,27 +190,3 @@ fn resolve_model_hook_entries<'py>( let resolution = Resolution::of_context(adapter, &context, external_sql_reference_resolver)?; model_hook_entries(model, &resolution) } - -/// One model's pre and post hooks with their SQL resolved. -pub(crate) fn model_hook_entries<'py>( - model: &Bound<'py, PyAny>, - resolution: &Resolution<'py>, -) -> PyResult> { - let py: Python<'py> = model.py(); - let values = model_values(model)?; - let name: String = model_name(model)?; - let mut hooks: Vec> = Vec::new(); - for key in ["pre_hooks", "post_hooks"] { - hooks.push(resolution.hook( - config_value(py, &values, key)?, - &format!("Model '{name}' {key}"), - )?); - } - PyTuple::new(model.py(), hooks) -} - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_model_sql_of, module)?)?; - module.add_function(wrap_pyfunction!(resolve_function_sql, module)?)?; - module.add_function(wrap_pyfunction!(resolve_model_hook_entries, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs similarity index 80% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs index 8429bcf5ba..23aff09270 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs @@ -2,14 +2,14 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; -use crate::bindings::_helpers::planner_resolution::values::{ +use crate::planner_resolution::_helpers::references::values::{ cursor_wrap, filtering_of, method, string, }; +use crate::planner_resolution::models::CursorFiltering; use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -20,7 +20,10 @@ use sqlbuild_sqltext::resolution::main::{ref_alias::ref_alias, reference_spans:: #[pyfunction] #[pyo3(signature = (**kwargs))] -fn resolve_ref_references(py: Python<'_>, kwargs: Option>) -> PyResult { +pub(crate) fn resolve_ref_references( + py: Python<'_>, + kwargs: Option>, +) -> PyResult { let args = KeywordArguments::read( py, "resolve_ref_references", @@ -228,40 +231,3 @@ pub(crate) fn resolve_table_function_references( ) -> PyResult { resolve_function_calls(query_sql, function_locations, adapter, true) } - -#[pyfunction] -#[pyo3(signature = (*, model_locations, seed_locations, deferred_locations, selected_keys))] -pub(crate) fn apply_deferred_locations<'py>( - model_locations: &Bound<'py, PyDict>, - seed_locations: &Bound<'py, PyDict>, - deferred_locations: &Bound<'py, PyDict>, - selected_keys: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { - let selected = selected_keys - .try_iter()? - .map(|key| native_field(&key?, "name")?.extract::()) - .collect::>>()?; - let models = model_locations.copy()?; - let seeds = seed_locations.copy()?; - for (name, target) in deferred_locations.iter() { - if selected.contains(&name.extract::()?) { - continue; - } - if models.contains(&name)? { - models.set_item(&name, &target)?; - } - if seeds.contains(&name)? { - seeds.set_item(&name, &target)?; - } - } - Ok((models, seeds)) -} - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_ref_references, module)?)?; - module.add_function(wrap_pyfunction!(resolve_dbt_ref_references, module)?)?; - module.add_function(wrap_pyfunction!(resolve_udf_references, module)?)?; - module.add_function(wrap_pyfunction!(resolve_table_function_references, module)?)?; - module.add_function(wrap_pyfunction!(apply_deferred_locations, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs index 10bf24d5ed..a650168767 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs @@ -1,16 +1,14 @@ //! Source contract validation and adapter-controlled projection rendering. -use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; -use crate::bindings::_helpers::planner_resolution::values::{ +use crate::planner_resolution::_helpers::references::values::{ cursor_wrap, filtering_of, method, string, }; -use crate::bindings::_helpers::planner_strategy::values::input_error; +use crate::planner_resolution::models::CursorFiltering; +use crate::planner_strategy::main::values::input_error::input_error; use pyo3::IntoPyObject; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; @@ -286,7 +284,7 @@ fn resolve_one( #[pyfunction] #[pyo3(signature = (**kwargs))] -fn resolve_source_references( +pub(crate) fn resolve_source_references( py: Python<'_>, kwargs: Option>, ) -> PyResult { @@ -351,9 +349,3 @@ pub(crate) fn resolve_sources( output.push_str(&sql[last..]); Ok(output) } - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_source_references, module)?)?; - module.add_function(wrap_pyfunction!(render_source_read_relation, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs index d2d5e16d63..689cb59b7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs @@ -1,6 +1,6 @@ //! Boundary calls shared by reference and source resolution. -use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; +use crate::planner_resolution::models::CursorFiltering; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyString; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -11,11 +11,11 @@ use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to use sqlbuild_sqltext::resolution::main::cursor_subquery::cursor_subquery; use sqlbuild_sqltext::resolution::models::CursorFilter; -pub(super) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { +pub(crate) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { PyString::new(py, text).into_any() } -pub(super) fn method<'py>( +pub(crate) fn method<'py>( object: &Bound<'py, PyAny>, name: &str, values: &[(&str, Bound<'py, PyAny>)], @@ -23,7 +23,7 @@ pub(super) fn method<'py>( call_with(&native_field(object, name)?, values) } -pub(super) fn cursor_wrap( +pub(crate) fn cursor_wrap( adapter: &Bound<'_, PyAny>, filtering: &CursorFiltering<'_>, relation: String, @@ -93,7 +93,7 @@ fn cursor_operand( } /// The cursor filtering of a keyword-argument resolution request. -pub(super) fn filtering_of<'py>( +pub(crate) fn filtering_of<'py>( arguments: &KeywordArguments<'py>, ) -> PyResult> { Ok(CursorFiltering { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/future_safety.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/future_safety.rs index f2dbdb5769..7de00a6118 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/future_safety.rs @@ -1,11 +1,11 @@ //! Future cursor safety evidence and exception boundary. -use crate::bindings::_helpers::planner_resolution::cursor_safety::{ +use crate::planner_resolution::_helpers::cursor::cursor_safety::{ horizon, replace, safety_duration, safety_error, timestamp_bound, utc_clock, }; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::main::next_boundary::next_boundary; use sqlbuild_cursor::algebra::main::render::render; @@ -163,7 +163,3 @@ pub(crate) fn resolve_future_cursor_safety<'py>( )?; replace(&bounds, &[("end", applied_end), ("future_safety", safety)]) } - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_future_cursor_safety, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs similarity index 82% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs rename to crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs index faa31a2276..ff3fd29285 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs @@ -1,20 +1,17 @@ //! Maximum automatic-start recovery and evidence boundary. -use crate::bindings::_helpers::planner_resolution::cursor_bounds::apply_typed_cursor_replay_policy; -use crate::bindings::_helpers::planner_resolution::cursor_bounds::{ +use crate::planner_resolution::_helpers::cursor::cursor_bounds::apply_typed_cursor_replay_policy; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::{ optional_scalar, required_scalar, scalars, }; -use crate::bindings::_helpers::planner_resolution::cursor_safety::{ +use crate::planner_resolution::_helpers::cursor::cursor_safety::{ horizon, replace, safety_duration, safety_error, timestamp_bound, utc_clock, }; -use crate::bindings::_helpers::planner_resolution::values::string; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use crate::planner_resolution::_helpers::references::values::string; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyDict; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_cursor::algebra::main::compare::compare; -use sqlbuild_cursor::algebra::main::render::render; use sqlbuild_cursor::planner::main::eligible_start::eligible_start; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -30,47 +27,6 @@ use sqlbuild_python_front::cursor::models::CursorSession; use sqlbuild_python_project::planner_phases::main::object::object; use std::cmp::Ordering; -#[pyfunction] -#[pyo3(signature = (**kwargs))] -pub(crate) fn maximum_allowed_start<'py>( - py: Python<'py>, - kwargs: Option>, -) -> PyResult { - let args = KeywordArguments::read( - py, - "maximum_allowed_start", - kwargs, - ( - &[ - "discovered_value", - "cursor_grain", - "invocation_time", - "max_ahead", - ], - &[], - ), - )?; - let session = CursorSession::new(py)?; - let discovered = required_scalar( - &session, - &args.value("discovered_value")?, - &string(py, "timestamp"), - )?; - let duration = safety_duration(&args.value("max_ahead")?, false)?; - session.checked(render( - &horizon( - &session, - &session.python_scalar(&discovered)?, - ( - &args.value("invocation_time")?, - &args.value("cursor_grain")?, - ), - &duration, - )?, - &session, - )) -} - #[pyfunction] #[pyo3(signature = (**kwargs))] pub(crate) fn apply_maximum_start_policy<'py>( @@ -284,8 +240,3 @@ fn exceeded_message(evidence: &Bound<'_, PyAny>, non_idempotent: bool) -> PyResu native_field(evidence, "cursor_column")?.str()?, )) } - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(maximum_allowed_start, module)?)?; - module.add_function(wrap_pyfunction!(apply_maximum_start_policy, module)?) -} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/mod.rs new file mode 100644 index 0000000000..f07ca613f3 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod future_safety; +pub(crate) mod maximum_safety; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/constants.rs b/crates/sqlbuild-python-planner/src/planner_resolution/constants.rs new file mode 100644 index 0000000000..35c1634370 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/constants.rs @@ -0,0 +1,3 @@ +//! Public constants of planner resolution. + +pub(crate) const CURSOR_START: &str = "cursor_start"; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/config_str.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/config_str.rs new file mode 100644 index 0000000000..673dbe2d6d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/config_str.rs @@ -0,0 +1,12 @@ +//! One string-valued model configuration entry. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; + +/// One string-valued model configuration entry. +pub fn config_str<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { + let py: Python<'py> = model.py(); + Ok(config_text_object(py, &model_values(model)?, key)? + .unwrap_or_else(|| py.None().into_bound(py))) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/cursor_safety_configs.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/cursor_safety_configs.rs new file mode 100644 index 0000000000..814d8fdea0 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/cursor_safety_configs.rs @@ -0,0 +1,25 @@ +//! The future-end and automatic-start policies effective for one model. + +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_phases::main::inputs::class_field::class_field; +use crate::planner_resolution::_helpers::cursor::cursor_policies::{ + future_cursor_config, start_cursor_config, +}; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// The future-end and automatic-start policies effective for one model. +pub fn cursor_safety_configs<'py>( + model: &Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + Ok(( + future_cursor_config( + model, + &class_field(context, |c: &ModelPlanContext| &c.future_cursor_config)?, + )?, + start_cursor_config( + model, + &class_field(context, |c: &ModelPlanContext| &c.start_cursor_config)?, + )?, + )) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/empty_cursor_inputs.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/empty_cursor_inputs.rs new file mode 100644 index 0000000000..8ca3bef534 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/empty_cursor_inputs.rs @@ -0,0 +1,39 @@ +//! Empty inputs that leave a planner-owned window empty, and whether others have rows. + +use crate::planner_resolution::_helpers::cursor::cursor_ranges::INCREMENTAL; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::ROLLING_WINDOW; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::model_cursor_snapshot; +use crate::planner_resolution::_helpers::cursor::cursor_windows::cursor_inputs_without_rows; +use crate::planner_resolution::main::cursor::config_str::config_str; +use crate::planner_resolution::models::CursorWindowInputs; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Empty inputs that leave a planner-owned window empty, and whether others have rows. +pub fn empty_cursor_inputs<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, +) -> PyResult<(Bound<'py, PyAny>, bool)> { + let py = model.py(); + let nothing = + || -> PyResult<(Bound<'py, PyAny>, bool)> { Ok((PyTuple::empty(py).into_any(), false)) }; + if inputs.full_refresh + || inputs.runtime_owned + || !config_str(model, "materialized")?.eq(INCREMENTAL)? + || config_str(model, "cursor")?.is_none() + || config_str(model, "microbatch_strategy")?.eq(ROLLING_WINDOW)? + || (!inputs.overrides.0.is_none() && !inputs.overrides.1.is_none()) + { + return nothing(); + } + let cursor_snapshot = model_cursor_snapshot(model, &inputs.snapshot)?; + if cursor_snapshot.is_none() { + return nothing(); + } + let decision = cursor_inputs_without_rows(&cursor_snapshot)?; + Ok(( + native_field(&decision, "input_names")?, + native_field(&decision, "waiting_on_empty_inputs")?.is_truthy()?, + )) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/microbatch_range.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/microbatch_range.rs new file mode 100644 index 0000000000..86132f1819 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/microbatch_range.rs @@ -0,0 +1,14 @@ +//! Compute the real overall cursor range for microbatch batch splitting. + +use crate::planner_resolution::_helpers::cursor::cursor_ranges; +use crate::planner_resolution::models::CursorWindowInputs; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Compute the real overall cursor range for microbatch batch splitting. +#[inline] +pub fn microbatch_range<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, +) -> PyResult> { + cursor_ranges::microbatch_range(model, inputs) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/mod.rs new file mode 100644 index 0000000000..221a4d8047 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/mod.rs @@ -0,0 +1,16 @@ +pub mod config_str; +pub mod cursor_safety_configs; +pub mod empty_cursor_inputs; +pub mod microbatch_range; +pub mod model_cursor_overrides; +pub mod normalize_cursor_snapshot_grain; +pub mod plan_cursor_bounds; +pub mod register_cursor_bounds; +pub mod register_cursor_overrides; +pub mod register_cursor_ranges; +pub mod register_cursor_warnings; +pub mod register_cursor_windows; +pub mod resolve_effective_timestamp_grain; +pub mod resolve_start_cursor_config; +pub mod type_cursor_overrides; +pub mod with_cursor_safety_warnings; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/model_cursor_overrides.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/model_cursor_overrides.rs new file mode 100644 index 0000000000..12a86e1137 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/model_cursor_overrides.rs @@ -0,0 +1,78 @@ +//! Resolve typed cursor overrides to generic start/end for one model. + +use crate::planner_resolution::_helpers::cursor::cursor_ranges::INTEGER; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::TIMESTAMP; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::compile_input_error; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::config_bound; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::order; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::token; +use crate::planner_resolution::main::cursor::config_str::config_str; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::cursor::models::CursorSession; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use std::cmp::Ordering; + +/// Resolve typed cursor overrides to generic start/end for one model. +pub fn model_cursor_overrides<'py>( + model: &Bound<'py, PyAny>, + cursor_overrides: &Bound<'py, PyAny>, + (start, end): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py = model.py(); + let cursor_type = config_str(model, "cursor_type")?; + let (mut resolved_start, mut resolved_end) = (start, end); + let typed_fields = if cursor_overrides.is_none() { + None + } else if cursor_type.eq(TIMESTAMP)? { + Some(("start_ts", "end_ts")) + } else if cursor_type.eq(INTEGER)? { + Some(("start_int", "end_int")) + } else { + None + }; + if let Some((start_field, end_field)) = typed_fields { + let typed_start = native_field(cursor_overrides, start_field)?; + if !typed_start.is_none() { + resolved_start = token(py, &typed_start)?; + } + let typed_end = native_field(cursor_overrides, end_field)?; + if !typed_end.is_none() { + resolved_end = token(py, &typed_end)?; + } + } + let session = CursorSession::new(py)?; + let name = name_object(model)?.str()?; + let cursor_start = config_bound(model, "cursor_start")?; + let cursor_end = config_bound(model, "cursor_end")?; + if !resolved_start.is_none() + && !cursor_start.is_none() + && order(&session, (&resolved_start, &cursor_start), &cursor_type)? == Ordering::Less + { + return Err(compile_input_error( + py, + format!( + "model '{name}': explicit start {} is before cursor_start {}", + resolved_start.str()?, + cursor_start.str()? + ), + )); + } + if !cursor_end.is_none() { + for (value, label) in [(&resolved_start, "start"), (&resolved_end, "end")] { + if !value.is_none() + && order(&session, (value, &cursor_end), &cursor_type)? != Ordering::Less + { + return Err(compile_input_error( + py, + format!( + "model '{name}': explicit {label} {} is at or beyond exclusive cursor_end {}", + value.str()?, + cursor_end.str()? + ), + )); + } + } + } + Ok((resolved_start, resolved_end)) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/normalize_cursor_snapshot_grain.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/normalize_cursor_snapshot_grain.rs new file mode 100644 index 0000000000..e9465f70cc --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/normalize_cursor_snapshot_grain.rs @@ -0,0 +1,14 @@ +//! Normalize cursor snapshot grain. + +use crate::planner_resolution::_helpers::cursor::cursor_windows; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Normalize cursor snapshot grain. +#[inline] +pub fn normalize_cursor_snapshot_grain<'py>( + cursor_snapshot: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, + effective_grain: &Bound<'py, PyAny>, +) -> PyResult> { + cursor_windows::normalize_cursor_snapshot_grain(cursor_snapshot, cursor_type, effective_grain) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/plan_cursor_bounds.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/plan_cursor_bounds.rs new file mode 100644 index 0000000000..050b6e8cb9 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/plan_cursor_bounds.rs @@ -0,0 +1,55 @@ +//! Compute cursor bounds for inclusion on the plan entry. + +use crate::planner_resolution::_helpers::cursor::cursor_ranges::BoundsRequest; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::FULL_BACKFILL; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::INCREMENTAL; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::MICROBATCH; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::bounded_override; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::clamp_cursor_end; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::model_cursor_snapshot; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::snapshot_bounds; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::without_destination; +use crate::planner_resolution::main::cursor::config_str::config_str; +use crate::planner_resolution::models::CursorWindowInputs; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Compute cursor bounds for inclusion on the plan entry. +pub fn plan_cursor_bounds<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, +) -> PyResult> { + let py = model.py(); + if !config_str(model, "materialized")?.eq(INCREMENTAL)? + || config_str(model, "cursor")?.is_none() + { + return Ok(py.None().into_bound(py)); + } + let bounded = bounded_override(model, inputs)?; + if !bounded.is_none() { + return clamp_cursor_end(bounded, model); + } + if inputs.runtime_owned || inputs.full_refresh { + return Ok(py.None().into_bound(py)); + } + let mut cursor_snapshot = model_cursor_snapshot(model, &inputs.snapshot)?; + if cursor_snapshot.is_none() { + return Ok(cursor_snapshot); + } + if native_field(&inputs.backfill, "action")?.eq(FULL_BACKFILL)? { + cursor_snapshot = without_destination(cursor_snapshot)?; + } + snapshot_bounds( + model, + inputs, + BoundsRequest { + cursor_snapshot, + typing: ( + config_str(model, "cursor_type")?, + config_str(model, "cursor_grain")?, + ), + lookback: config_str(model, "lookback")?, + is_microbatch: config_str(model, "incremental_mode")?.eq(MICROBATCH)?, + }, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_bounds.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_bounds.rs new file mode 100644 index 0000000000..8b84cfd621 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_bounds.rs @@ -0,0 +1,12 @@ +//! Register the cursor bounds functions and classes. + +use crate::planner_resolution::_helpers::cursor::cursor_bounds::apply_typed_cursor_replay_policy; +use crate::planner_resolution::_helpers::cursor::cursor_bounds::compute_cursor_bounds; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the cursor bounds functions and classes. +pub fn register_cursor_bounds(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compute_cursor_bounds, module)?)?; + module.add_function(wrap_pyfunction!(apply_typed_cursor_replay_policy, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_overrides.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_overrides.rs new file mode 100644 index 0000000000..78ed64fb26 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_overrides.rs @@ -0,0 +1,10 @@ +//! Register the cursor overrides functions and classes. + +use crate::planner_resolution::_helpers::cursor::cursor_overrides::type_cursor_overrides_of; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the cursor overrides functions and classes. +pub fn register_cursor_overrides(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(type_cursor_overrides_of, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs new file mode 100644 index 0000000000..ee4dee1ca0 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs @@ -0,0 +1,14 @@ +//! Register the cursor ranges functions and classes. + +use crate::planner_resolution::_helpers::cursor::cursor_ranges::compute_microbatch_range; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::compute_plan_cursor_bounds; +use crate::planner_resolution::_helpers::cursor::cursor_ranges::resolve_cursor_overrides; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the cursor ranges functions and classes. +pub fn register_cursor_ranges(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compute_plan_cursor_bounds, module)?)?; + module.add_function(wrap_pyfunction!(compute_microbatch_range, module)?)?; + module.add_function(wrap_pyfunction!(resolve_cursor_overrides, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_warnings.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_warnings.rs new file mode 100644 index 0000000000..7680ec73cc --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_warnings.rs @@ -0,0 +1,12 @@ +//! Register the cursor warnings functions and classes. + +use crate::planner_resolution::_helpers::cursor::cursor_warnings::future_cursor_cap_warning; +use crate::planner_resolution::_helpers::cursor::cursor_warnings::maximum_start_cap_warning; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the cursor warnings functions and classes. +pub fn register_cursor_warnings(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(future_cursor_cap_warning, module)?)?; + module.add_function(wrap_pyfunction!(maximum_start_cap_warning, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_windows.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_windows.rs new file mode 100644 index 0000000000..bb188ec60e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_windows.rs @@ -0,0 +1,20 @@ +//! Register the cursor windows functions and classes. + +use crate::planner_resolution::_helpers::cursor::cursor_windows::cursor_window_override_flags; +use crate::planner_resolution::_helpers::cursor::cursor_windows::decide_empty_cursor_inputs; +use crate::planner_resolution::_helpers::cursor::cursor_windows::empty_cursor_window; +use crate::planner_resolution::_helpers::cursor::cursor_windows::empty_input_rebuild_refusal; +use crate::planner_resolution::_helpers::cursor::cursor_windows::normalize_cursor_snapshot_grain; +use crate::planner_resolution::main::cursor::resolve_effective_timestamp_grain::resolve_effective_timestamp_grain; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the cursor windows functions and classes. +pub fn register_cursor_windows(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(decide_empty_cursor_inputs, module)?)?; + module.add_function(wrap_pyfunction!(empty_cursor_window, module)?)?; + module.add_function(wrap_pyfunction!(cursor_window_override_flags, module)?)?; + module.add_function(wrap_pyfunction!(empty_input_rebuild_refusal, module)?)?; + module.add_function(wrap_pyfunction!(resolve_effective_timestamp_grain, module)?)?; + module.add_function(wrap_pyfunction!(normalize_cursor_snapshot_grain, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_effective_timestamp_grain.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_effective_timestamp_grain.rs new file mode 100644 index 0000000000..18e239e2d7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_effective_timestamp_grain.rs @@ -0,0 +1,41 @@ +//! Resolve effective timestamp grain. + +use crate::planner_resolution::_helpers::references::values::string; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_front::cursor::models::CursorSession; + +#[pyfunction] +#[pyo3(signature = (*, cursor_type, downstream_grain, cursor_input_grains, microbatch_strategy=None))] +pub fn resolve_effective_timestamp_grain<'py>( + cursor_type: &Bound<'py, PyAny>, + downstream_grain: &Bound<'py, PyAny>, + cursor_input_grains: &Bound<'py, PyAny>, + microbatch_strategy: Option<&str>, +) -> PyResult> { + let py = cursor_type.py(); + if !cursor_type.eq("timestamp")? { + return Ok(py.None().into_bound(py)); + } + let mut effective = if downstream_grain.is_truthy()? { + downstream_grain.clone() + } else { + string(py, "second") + }; + if microbatch_strategy == Some("watermark") { + return Ok(effective); + } + let session = CursorSession::new(py)?; + for value in cursor_input_grains.try_iter()? { + let value = value?; + let candidate = if value.is_truthy()? { + value + } else { + string(py, "second") + }; + if session.grain(&candidate)? > session.grain(&effective)? { + effective = candidate; + } + } + Ok(effective) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_start_cursor_config.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_start_cursor_config.rs new file mode 100644 index 0000000000..2e6a9c6002 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/resolve_start_cursor_config.rs @@ -0,0 +1,12 @@ +//! Resolve automatic-start policy with model config taking precedence. + +use crate::planner_resolution::_helpers::cursor::cursor_policies::start_cursor_config; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Resolve automatic-start policy with model config taking precedence. +pub fn resolve_start_cursor_config<'py>( + model: &Bound<'py, PyAny>, + project_config: &Bound<'py, PyAny>, +) -> PyResult> { + start_cursor_config(model, project_config) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/type_cursor_overrides.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/type_cursor_overrides.rs new file mode 100644 index 0000000000..9d377e504c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/type_cursor_overrides.rs @@ -0,0 +1,13 @@ +//! Untyped cursor overrides typed by the cursor type of the selected incremental models. + +use crate::planner_resolution::_helpers::cursor::cursor_overrides; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Untyped cursor overrides typed by the cursor type of the selected incremental models. +#[inline] +pub fn type_cursor_overrides<'py>( + overrides: &Bound<'py, PyAny>, + selected: &[Bound<'py, PyAny>], +) -> PyResult> { + cursor_overrides::type_cursor_overrides(overrides, selected) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/with_cursor_safety_warnings.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/with_cursor_safety_warnings.rs new file mode 100644 index 0000000000..0f6a337dc8 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/with_cursor_safety_warnings.rs @@ -0,0 +1,44 @@ +//! `warnings` followed by the future-end and automatic-start warnings `bounds` carries. + +use crate::planner_resolution::_helpers::cursor::cursor_warnings::FUTURE_CAPPED_CODE; +use crate::planner_resolution::_helpers::cursor::cursor_warnings::START_CAPPED_CODE; +use crate::planner_resolution::_helpers::cursor::cursor_warnings::future_cap_message; +use crate::planner_resolution::_helpers::cursor::cursor_warnings::maximum_start_message; +use crate::planner_resolution::_helpers::references::values::string; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; + +/// `warnings` followed by the future-end and automatic-start warnings `bounds` carries. +pub fn with_cursor_safety_warnings<'py>( + warnings: &Bound<'py, PyAny>, + model_name: &Bound<'py, PyAny>, + bounds: &Bound<'py, PyAny>, +) -> PyResult> { + let py = warnings.py(); + let mut all: Vec> = warnings + .try_iter()? + .collect::>>>()?; + for (message, code) in [ + (future_cap_message(bounds)?, FUTURE_CAPPED_CODE), + (maximum_start_message(bounds)?, START_CAPPED_CODE), + ] { + let Some(message) = message else { continue }; + all.push(new_record_with( + py, + RecordId::PlanWarning, + &[ + ("model_name", model_name.clone()), + ( + "severity", + enum_member(py, RecordId::WarningSeverity, "WARNING")?, + ), + ("code", string(py, code)), + ("message", string(py, &message)), + ], + )?); + } + Ok(PyTuple::new(py, all)?.into_any()) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/mod.rs new file mode 100644 index 0000000000..be6ead50c4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/mod.rs @@ -0,0 +1,3 @@ +pub mod cursor; +pub mod references; +pub mod safety; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/apply_deferred_locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/apply_deferred_locations.rs new file mode 100644 index 0000000000..02f0a9ddea --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/apply_deferred_locations.rs @@ -0,0 +1,34 @@ +//! Apply deferred locations. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +#[pyfunction] +#[pyo3(signature = (*, model_locations, seed_locations, deferred_locations, selected_keys))] +pub fn apply_deferred_locations<'py>( + model_locations: &Bound<'py, PyDict>, + seed_locations: &Bound<'py, PyDict>, + deferred_locations: &Bound<'py, PyDict>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { + let selected = selected_keys + .try_iter()? + .map(|key| native_field(&key?, "name")?.extract::()) + .collect::>>()?; + let models = model_locations.copy()?; + let seeds = seed_locations.copy()?; + for (name, target) in deferred_locations.iter() { + if selected.contains(&name.extract::()?) { + continue; + } + if models.contains(&name)? { + models.set_item(&name, &target)?; + } + if seeds.contains(&name)? { + seeds.set_item(&name, &target)?; + } + } + Ok((models, seeds)) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_function_locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_function_locations.rs new file mode 100644 index 0000000000..19045ab702 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_function_locations.rs @@ -0,0 +1,13 @@ +//! Build function locations. + +use crate::planner_resolution::_helpers::references::locations::locations; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyDict; + +#[pyfunction] +pub fn build_function_locations<'py>( + functions: &Bound<'py, PyAny>, +) -> PyResult> { + locations(functions) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_model_locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_model_locations.rs new file mode 100644 index 0000000000..f71e189c57 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_model_locations.rs @@ -0,0 +1,11 @@ +//! Build model locations. + +use crate::planner_resolution::_helpers::references::locations::locations; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyDict; + +#[pyfunction] +pub fn build_model_locations<'py>(models: &Bound<'py, PyAny>) -> PyResult> { + locations(models) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_seed_locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_seed_locations.rs new file mode 100644 index 0000000000..68a222d3cb --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/build_seed_locations.rs @@ -0,0 +1,11 @@ +//! Build seed locations. + +use crate::planner_resolution::_helpers::references::locations::locations; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyDict; + +#[pyfunction] +pub fn build_seed_locations<'py>(seeds: &Bound<'py, PyAny>) -> PyResult> { + locations(seeds) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/mod.rs new file mode 100644 index 0000000000..ebc871f35e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/mod.rs @@ -0,0 +1,11 @@ +pub mod apply_deferred_locations; +pub mod build_function_locations; +pub mod build_model_locations; +pub mod build_seed_locations; +pub mod model_hook_entries; +pub mod register_locations; +pub mod register_orchestration; +pub mod register_refs; +pub mod register_sources; +pub mod resolve_function_sql; +pub mod resolve_model_sql; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/model_hook_entries.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/model_hook_entries.rs new file mode 100644 index 0000000000..1a1c3e3c93 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/model_hook_entries.rs @@ -0,0 +1,26 @@ +//! One model's pre and post hooks with their SQL resolved. + +use crate::planner_resolution::models::Resolution; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; + +/// One model's pre and post hooks with their SQL resolved. +pub fn model_hook_entries<'py>( + model: &Bound<'py, PyAny>, + resolution: &Resolution<'py>, +) -> PyResult> { + let py: Python<'py> = model.py(); + let values = model_values(model)?; + let name: String = model_name(model)?; + let mut hooks: Vec> = Vec::new(); + for key in ["pre_hooks", "post_hooks"] { + hooks.push(resolution.hook( + config_value(py, &values, key)?, + &format!("Model '{name}' {key}"), + )?); + } + PyTuple::new(model.py(), hooks) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs new file mode 100644 index 0000000000..edd12f9299 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs @@ -0,0 +1,14 @@ +//! Register the locations functions and classes. + +use crate::planner_resolution::main::references::build_function_locations::build_function_locations; +use crate::planner_resolution::main::references::build_model_locations::build_model_locations; +use crate::planner_resolution::main::references::build_seed_locations::build_seed_locations; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the locations functions and classes. +pub fn register_locations(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_model_locations, module)?)?; + module.add_function(wrap_pyfunction!(build_seed_locations, module)?)?; + module.add_function(wrap_pyfunction!(build_function_locations, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs new file mode 100644 index 0000000000..3934e61676 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs @@ -0,0 +1,14 @@ +//! Register the orchestration functions and classes. + +use crate::planner_resolution::_helpers::references::orchestration::resolve_model_hook_entries; +use crate::planner_resolution::_helpers::references::orchestration::resolve_model_sql_of; +use crate::planner_resolution::main::references::resolve_function_sql::resolve_function_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the orchestration functions and classes. +pub fn register_orchestration(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_model_sql_of, module)?)?; + module.add_function(wrap_pyfunction!(resolve_function_sql, module)?)?; + module.add_function(wrap_pyfunction!(resolve_model_hook_entries, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs new file mode 100644 index 0000000000..214f231b20 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs @@ -0,0 +1,19 @@ +//! Register the refs functions and classes. + +use crate::planner_resolution::_helpers::references::refs::resolve_dbt_ref_references; +use crate::planner_resolution::_helpers::references::refs::resolve_ref_references; +use crate::planner_resolution::_helpers::references::refs::resolve_table_function_references; +use crate::planner_resolution::_helpers::references::refs::resolve_udf_references; +use crate::planner_resolution::main::references::apply_deferred_locations::apply_deferred_locations; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the refs functions and classes. +pub fn register_refs(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_ref_references, module)?)?; + module.add_function(wrap_pyfunction!(resolve_dbt_ref_references, module)?)?; + module.add_function(wrap_pyfunction!(resolve_udf_references, module)?)?; + module.add_function(wrap_pyfunction!(resolve_table_function_references, module)?)?; + module.add_function(wrap_pyfunction!(apply_deferred_locations, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_sources.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_sources.rs new file mode 100644 index 0000000000..5fba9ca4f4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_sources.rs @@ -0,0 +1,13 @@ +//! Register the sources functions and classes. + +use crate::planner_resolution::_helpers::references::sources::render_source_read_relation; +use crate::planner_resolution::_helpers::references::sources::resolve_source_references; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the sources functions and classes. +pub fn register_sources(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_source_references, module)?)?; + module.add_function(wrap_pyfunction!(render_source_read_relation, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_function_sql.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_function_sql.rs new file mode 100644 index 0000000000..24046a923e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_function_sql.rs @@ -0,0 +1,49 @@ +//! Resolve function sql. + +use crate::planner_resolution::models::Resolution; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyString}; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub fn resolve_function_sql<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "resolve_function_sql", + kwargs, + ( + &[ + "adapter", + "function", + "model_locations", + "seed_locations", + "function_locations", + "source_map", + "source_warehouse_columns", + "star_exclude_keyword", + ], + &[], + ), + )?; + let resolution = Resolution { + adapter: args.value("adapter")?, + external: py.None().into_bound(py), + source_map: args.value("source_map")?, + source_warehouse_columns: args.value("source_warehouse_columns")?, + model_locations: args.value("model_locations")?, + seed_locations: args.value("seed_locations")?, + function_locations: args.value("function_locations")?, + }; + let function = args.value("function")?; + let name: String = + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)? + .extract()?; + let sql: String = resolution.unbounded(body_sql(&function)?, &format!("Function '{name}'"))?; + Ok(PyString::new(py, &sql).into_any()) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_model_sql.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_model_sql.rs new file mode 100644 index 0000000000..2747df8f56 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/references/resolve_model_sql.rs @@ -0,0 +1,16 @@ +//! One model's planned SQL: references resolved, cursor inputs filtered and intrinsics bound. + +use crate::planner_resolution::_helpers::references::orchestration; +use crate::planner_resolution::models::ModelSqlRequest; +use crate::planner_resolution::models::Resolution; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// One model's planned SQL: references resolved, cursor inputs filtered and intrinsics bound. +#[inline] +pub fn resolve_model_sql<'py>( + resolution: &Resolution<'py>, + context: &Bound<'py, PyAny>, + request: &ModelSqlRequest<'py>, +) -> PyResult { + orchestration::resolve_model_sql(resolution, context, request) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/maximum_allowed_start.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/maximum_allowed_start.rs new file mode 100644 index 0000000000..caaae5d68c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/maximum_allowed_start.rs @@ -0,0 +1,52 @@ +//! Maximum allowed start. + +use crate::planner_resolution::_helpers::cursor::cursor_bounds::required_scalar; +use crate::planner_resolution::_helpers::cursor::cursor_safety::{horizon, safety_duration}; +use crate::planner_resolution::_helpers::references::values::string; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_cursor::algebra::main::render::render; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_front::cursor::models::CursorSession; + +#[pyfunction] +#[pyo3(signature = (**kwargs))] +pub fn maximum_allowed_start<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult { + let args = KeywordArguments::read( + py, + "maximum_allowed_start", + kwargs, + ( + &[ + "discovered_value", + "cursor_grain", + "invocation_time", + "max_ahead", + ], + &[], + ), + )?; + let session = CursorSession::new(py)?; + let discovered = required_scalar( + &session, + &args.value("discovered_value")?, + &string(py, "timestamp"), + )?; + let duration = safety_duration(&args.value("max_ahead")?, false)?; + session.checked(render( + &horizon( + &session, + &session.python_scalar(&discovered)?, + ( + &args.value("invocation_time")?, + &args.value("cursor_grain")?, + ), + &duration, + )?, + &session, + )) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/mod.rs new file mode 100644 index 0000000000..bac12ee9a3 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/mod.rs @@ -0,0 +1,3 @@ +pub mod maximum_allowed_start; +pub mod register_future_safety; +pub mod register_maximum_safety; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_future_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_future_safety.rs new file mode 100644 index 0000000000..4884879308 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_future_safety.rs @@ -0,0 +1,10 @@ +//! Register the future safety functions and classes. + +use crate::planner_resolution::_helpers::safety::future_safety::resolve_future_cursor_safety; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the future safety functions and classes. +pub fn register_future_safety(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_future_cursor_safety, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_maximum_safety.rs b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_maximum_safety.rs new file mode 100644 index 0000000000..7d50ed639c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/main/safety/register_maximum_safety.rs @@ -0,0 +1,12 @@ +//! Register the maximum safety functions and classes. + +use crate::planner_resolution::_helpers::safety::maximum_safety::apply_maximum_start_policy; +use crate::planner_resolution::main::safety::maximum_allowed_start::maximum_allowed_start; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the maximum safety functions and classes. +pub fn register_maximum_safety(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(maximum_allowed_start, module)?)?; + module.add_function(wrap_pyfunction!(apply_maximum_start_policy, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/mod.rs b/crates/sqlbuild-python-planner/src/planner_resolution/mod.rs new file mode 100644 index 0000000000..e8dd7a3eda --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/mod.rs @@ -0,0 +1,6 @@ +//! Planner reference resolution with Python adapter rendering callbacks. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/models.rs b/crates/sqlbuild-python-planner/src/planner_resolution/models.rs new file mode 100644 index 0000000000..7db929d5a7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_resolution/models.rs @@ -0,0 +1,167 @@ +//! Public records of planner resolution. + +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_phases::main::inputs::class_field::class_field; +use crate::planner_resolution::_helpers::references::refs::resolve_dbt_ref_references; +use crate::planner_resolution::_helpers::references::refs::resolve_refs; +use crate::planner_resolution::_helpers::references::refs::resolve_table_function_references; +use crate::planner_resolution::_helpers::references::refs::resolve_udf_references; +use crate::planner_resolution::_helpers::references::sources::resolve_sources; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny}; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyList; +use pyo3::types::PyString; +use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::attachments::main::assert_no_unresolved_sql_markers::assert_no_unresolved_sql_markers; + +/// The planner inputs that decide one model's plan-time cursor window. +#[derive(Debug)] +pub struct CursorWindowInputs<'py> { + pub snapshot: Bound<'py, PyAny>, + pub backfill: Bound<'py, PyAny>, + pub overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub runtime_owned: bool, + pub policies: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub invocation_time: Bound<'py, PyAny>, + pub full_refresh: bool, + pub replaces_relation: bool, + pub cursor_input_relations: Bound<'py, PyAny>, +} + +/// The cursor window that filters the cursor inputs one resolution reads. +#[derive(Debug)] +pub(crate) struct CursorFiltering<'py> { + pub bounds: Bound<'py, PyAny>, + pub filter_inputs: Bound<'py, PyAny>, + pub cursor_type: Bound<'py, PyAny>, + pub lower_bound_inclusive: bool, +} + +/// One model's SQL resolution decisions. +#[derive(Debug)] +pub struct ModelSqlRequest<'py> { + pub model: Bound<'py, PyAny>, + pub snapshot: Bound<'py, PyAny>, + pub backfill: Bound<'py, PyAny>, + pub full_refresh: bool, + pub cursor_overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub replaces_relation: bool, + pub full_rebuild_cause: Option, +} + +/// What resolving one model's, hook's or function's references reads. +#[derive(Debug)] +pub struct Resolution<'py> { + pub adapter: Bound<'py, PyAny>, + pub(crate) external: Bound<'py, PyAny>, + pub(crate) source_map: Bound<'py, PyAny>, + pub(crate) source_warehouse_columns: Bound<'py, PyAny>, + pub(crate) model_locations: Bound<'py, PyAny>, + pub(crate) seed_locations: Bound<'py, PyAny>, + pub(crate) function_locations: Bound<'py, PyAny>, +} + +impl<'py> Resolution<'py> { + /// Resolution against a `ModelPlanContext`'s relations. + pub fn of_context( + adapter: Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, + external: Bound<'py, PyAny>, + ) -> PyResult { + Ok(Self { + adapter, + external, + source_map: class_field(context, |c: &ModelPlanContext| &c.source_map)?, + source_warehouse_columns: class_field(context, |c: &ModelPlanContext| { + &c.source_warehouse_columns + })?, + model_locations: class_field(context, |c: &ModelPlanContext| &c.model_locations)?, + seed_locations: class_field(context, |c: &ModelPlanContext| &c.seed_locations)?, + function_locations: class_field(context, |c: &ModelPlanContext| &c.function_locations)?, + }) + } + + pub(crate) fn resolve(&self, sql: &str, filtering: &CursorFiltering<'py>) -> PyResult { + let py = self.adapter.py(); + let resolved: String = resolve_sources( + sql, + (&self.source_map, &self.source_warehouse_columns), + &self.adapter, + filtering, + )?; + let resolved: String = resolve_refs( + &resolved, + (&self.model_locations, &self.seed_locations), + &self.adapter, + filtering, + )?; + let resolved: String = resolve_dbt_ref_references(&resolved, Some(&self.external))?; + let locations = if self.function_locations.is_truthy()? { + self.function_locations.clone() + } else { + PyDict::new(py).into_any() + }; + let resolved: String = resolve_udf_references(&resolved, &locations, &self.adapter)?; + resolve_table_function_references(&resolved, &locations, &self.adapter) + } + + pub(crate) fn assert_resolved(&self, sql: &str, label: &str) -> PyResult<()> { + let py = self.adapter.py(); + assert_no_unresolved_sql_markers( + py, + &PyString::new(py, sql), + &format!("{label} planned SQL"), + ) + } + + pub(crate) fn unbounded(&self, sql: Bound<'py, PyAny>, label: &str) -> PyResult { + let py = self.adapter.py(); + let resolved = self.resolve( + &sql.extract::()?, + &CursorFiltering { + bounds: py.None().into_bound(py), + filter_inputs: PyDict::new(py).into_any(), + cursor_type: py.None().into_bound(py), + lower_bound_inclusive: true, + }, + )?; + self.assert_resolved(&resolved, label)?; + Ok(resolved) + } + + pub(crate) fn hook( + &self, + value: Bound<'py, PyAny>, + label: &str, + ) -> PyResult> { + let py = value.py(); + if value.is_instance(&record_class(py, RecordId::SqlHookEntry)?)? { + let kwargs = PyDict::new(py); + kwargs.set_item( + "statement", + self.unbounded(native_field(&value, "statement")?, label)?, + )?; + return replaced_keywords(&value, &kwargs); + } + let is_list = value.is_instance_of::(); + if is_list || value.is_instance_of::() { + let mut resolved: Vec> = Vec::new(); + for (index, item) in value.try_iter()?.enumerate() { + resolved.push(self.hook(item?, &format!("{label}[{index}]"))?); + } + return if is_list { + Ok(PyList::new(py, resolved)?.into_any()) + } else { + Ok(PyTuple::new(py, resolved)?.into_any()) + }; + } + Ok(value) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs index d941f2657e..ed30375b64 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs @@ -1,10 +1,10 @@ //! Scenario artifact dataclasses and adapter identifier callbacks. -use crate::bindings::_helpers::planner_scenario::constants::{ +use crate::planner_scenario::_helpers::values::{error, string}; +use crate::planner_scenario::constants::{ SCENARIO_PLAN_HASH_COLLISION, SCENARIO_PLAN_INVALID_HASH_PREFIX, SCENARIO_PLAN_RELATION_COLLISION, }; -use crate::bindings::_helpers::planner_scenario::values::{error, string}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_scenario::main::{hash_prefix, parse_artifact_name}; @@ -176,11 +176,6 @@ pub(crate) fn parse_scenario_artifact_physical_name<'py>( ) } -#[pyfunction] -pub(crate) fn is_scenario_artifact_physical_name(name: &str) -> bool { - parse_artifact_name::parse_artifact_name(name).is_some() -} - #[pyfunction] #[pyo3(signature = (**kwargs))] pub(crate) fn build_scenario_relation_map<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs index ab2e5b1c87..372837a2b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs @@ -1,13 +1,13 @@ //! Assemble CLI scenario plans while retaining adapter and executor boundaries. -use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_hash_index; -use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_relation_map; -use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; -use crate::bindings::_helpers::planner_scenario::execution::build_scenario_execution_plan; -use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; -use crate::bindings::_helpers::planner_scenario::models::ScenarioRelations; -use crate::bindings::_helpers::planner_scenario::relations::build_scenario_relation_plan; -use crate::bindings::_helpers::planner_scenario::values::string; +use crate::planner_scenario::_helpers::artifacts::build_scenario_hash_index; +use crate::planner_scenario::_helpers::artifacts::build_scenario_relation_map; +use crate::planner_scenario::_helpers::execution::build_scenario_execution_plan; +use crate::planner_scenario::_helpers::graph::plan_scenario_graph; +use crate::planner_scenario::_helpers::relations::build_scenario_relation_plan; +use crate::planner_scenario::_helpers::values::string; +use crate::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; +use crate::planner_scenario::models::ScenarioRelations; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs similarity index 88% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs index b26466437c..44e9e48e1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs @@ -1,12 +1,12 @@ //! Existing seed and function plan dataclass construction. -use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; -use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; -use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_UNKNOWN_SEED; -use crate::bindings::_helpers::planner_scenario::{ - models::SqlContext, sql::required_target, values::error, -}; -use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; +use crate::planner_identity::main::build_compiled_function_fingerprint_sql::build_compiled_function_fingerprint_sql; +use crate::planner_resolution::main::references::resolve_function_sql::resolve_function_sql; +use crate::planner_scenario::_helpers::sql::required_target; +use crate::planner_scenario::_helpers::values::error; +use crate::planner_scenario::constants::SCENARIO_PLAN_UNKNOWN_SEED; +use crate::planner_scenario::models::SqlContext; +use crate::planner_strategy::main::model_entries::extract_seed_columns::extract_seed_columns; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyString; use pyo3::types::{PyDictMethods, PyTuple}; @@ -19,7 +19,7 @@ use sqlbuild_python_project::resource_views::main::seed_reads::key::key; use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; -pub(super) fn seed_entries<'py>( +pub(crate) fn seed_entries<'py>( project: &Bound<'py, PyAny>, graph: &Bound<'py, PyAny>, relations: &Bound<'py, PyAny>, @@ -74,7 +74,7 @@ pub(super) fn seed_entries<'py>( PyTuple::new(py, result) } -pub(super) fn function_entry<'py>( +pub(crate) fn function_entry<'py>( function: &Bound<'py, PyAny>, context: &SqlContext<'py>, locations: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs index db3da0dc18..46c3b16a06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs @@ -1,21 +1,25 @@ //! Assemble scenario execution plans through existing planner and adapter APIs. -use crate::bindings::_helpers::planner_entries::model_entries::plan_model; -use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; -use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; -use crate::bindings::_helpers::planner_scenario::constants::{ +use crate::planner_entries::main::entries::plan_model::plan_model; +use crate::planner_entries::models::ModelPlanContext; +use crate::planner_resolution::main::references::build_function_locations::build_function_locations; +use crate::planner_scenario::_helpers::entries::function_entry; +use crate::planner_scenario::_helpers::entries::seed_entries; +use crate::planner_scenario::_helpers::fixtures::extract_fixtures; +use crate::planner_scenario::_helpers::fixtures::fixture_plans; +use crate::planner_scenario::_helpers::helper_scope::scenario_helper_scope; +use crate::planner_scenario::_helpers::sql::required_target; +use crate::planner_scenario::_helpers::sql::resolve_model_dbt_refs; +use crate::planner_scenario::_helpers::sql::resolve_sql; +use crate::planner_scenario::_helpers::sql::source_label; +use crate::planner_scenario::_helpers::values::error; +use crate::planner_scenario::_helpers::values::string; +use crate::planner_scenario::constants::{ CURSOR_INTRINSIC_NAMES, CURSOR_TYPE_KEY, INCREMENTAL_MODE_KEY, INTEGER_CURSOR_TYPE, MICROBATCH_INCREMENTAL_MODE, SCENARIO_EXPECTED_COLUMNS_HELP, SCENARIO_PLAN_EXPECTED_COLUMNS, SCENARIO_PLAN_INVALID_FIXTURE, }; -use crate::bindings::_helpers::planner_scenario::helper_scope::scenario_helper_scope; -use crate::bindings::_helpers::planner_scenario::{ - entries::{function_entry, seed_entries}, - fixtures::{extract_fixtures, fixture_plans}, - models::SqlContext, - sql::{required_target, resolve_model_dbt_refs, resolve_sql, source_label}, - values::{error, string}, -}; +use crate::planner_scenario::models::SqlContext; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::compiler::main::scenario_expected_column_message::scenario_expected_column_message; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/fixtures.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/fixtures.rs index 669fa0088e..32509765e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/fixtures.rs @@ -1,12 +1,13 @@ //! Materialize fixture SQL plans while preserving adapter and completion boundaries. -use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_MISSING_FIXTURE_SQL; -use crate::bindings::_helpers::planner_scenario::helper_scope::scenario_helper_scope; -use crate::bindings::_helpers::planner_scenario::{ - models::SqlContext, - sql::{required_target, resolve_sql, source_label}, - values::{error, string}, -}; +use crate::planner_scenario::_helpers::helper_scope::scenario_helper_scope; +use crate::planner_scenario::_helpers::sql::required_target; +use crate::planner_scenario::_helpers::sql::resolve_sql; +use crate::planner_scenario::_helpers::sql::source_label; +use crate::planner_scenario::_helpers::values::error; +use crate::planner_scenario::_helpers::values::string; +use crate::planner_scenario::constants::SCENARIO_PLAN_MISSING_FIXTURE_SQL; +use crate::planner_scenario::models::SqlContext; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::compiler::main::scenario_reader_ctes::scenario_reader_ctes; @@ -24,7 +25,7 @@ use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use std::collections::{BTreeMap, BTreeSet}; -pub(super) fn extract_fixtures<'py>( +pub(crate) fn extract_fixtures<'py>( scenario: &Bound<'py, PyAny>, prefix: &str, ) -> PyResult> { @@ -34,7 +35,7 @@ pub(super) fn extract_fixtures<'py>( ) } -pub(super) fn fixture_plans<'py>( +pub(crate) fn fixture_plans<'py>( scenario: &Bound<'py, PyAny>, graph: &Bound<'py, PyAny>, context: &SqlContext<'py>, @@ -77,7 +78,7 @@ pub(super) fn fixture_plans<'py>( )?); }; let mut sql = authored.clone(); - if kind != super::constants::DBT_REF && !overrides.is_none() { + if kind != crate::planner_scenario::constants::DBT_REF && !overrides.is_none() { sql = mapping_get_or(overrides, (resource, &name), sql)?; } let sql = resolve_sql( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/graph.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/graph.rs index e369c1a279..a1cb11e6d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/graph.rs @@ -1,7 +1,7 @@ //! Convert project views into pure scenario dependency facts. -use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; -use crate::bindings::_helpers::planner_scenario::values::string; +use crate::planner_scenario::_helpers::values::string; +use crate::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; use pyo3::types::PyTuple; use sqlbuild_analysis::planner_scenario::main::graph::graph; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/helper_scope.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/helper_scope.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/helper_scope.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/helper_scope.rs index 3b66e666b8..42a7e3491f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/helper_scope.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/helper_scope.rs @@ -4,8 +4,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::compiler::main::scenario_helper_scope::scenario_helper_scope as helper_scope; use sqlbuild_analysis::compiler::models::{ScenarioHelperScope, ScenarioScopeInput}; -use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_INTERNAL; -use crate::bindings::_helpers::planner_scenario::{models::SqlContext, values::error}; +use crate::planner_scenario::_helpers::values::error; +use crate::planner_scenario::constants::SCENARIO_PLAN_INTERNAL; +use crate::planner_scenario::models::SqlContext; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; @@ -18,7 +19,7 @@ const FIXTURE_LOCATIONS: [(&str, &str); 4] = [ ]; /// The scenario's helper scope, where a check reading a fixture by name reads its relation. -pub(super) fn scenario_helper_scope( +pub(crate) fn scenario_helper_scope( scenario: &Bound<'_, PyAny>, context: &SqlContext<'_>, ) -> PyResult { diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/mod.rs new file mode 100644 index 0000000000..9721320a6b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod artifacts; +pub(crate) mod cli; +pub(crate) mod entries; +pub(crate) mod execution; +pub(crate) mod fixtures; +pub(crate) mod graph; +pub(crate) mod helper_scope; +pub(crate) mod relations; +pub(crate) mod sql; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/relations.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/relations.rs index 02f6055a68..035c44aca4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/relations.rs @@ -1,11 +1,11 @@ //! Build scenario relation locations through adapter qualification callbacks. -use crate::bindings::_helpers::planner_scenario::constants::{REF, SOURCE}; -use crate::bindings::_helpers::planner_scenario::constants::{ +use crate::planner_scenario::_helpers::values::error; +use crate::planner_scenario::constants::{REF, SOURCE}; +use crate::planner_scenario::constants::{ SCENARIO_PLAN_INTERNAL, SCENARIO_PLAN_MISSING_RELATION_TARGET, }; -use crate::bindings::_helpers::planner_scenario::models::ScenarioRelations; -use crate::bindings::_helpers::planner_scenario::values::error; +use crate::planner_scenario::models::ScenarioRelations; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::boundary::main::call_with::call_with; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs index 323632ffd2..700030b8e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs @@ -1,12 +1,11 @@ //! Resolve scenario SQL through the shared native scanner and adapter callbacks. -use crate::bindings::_helpers::planner_scenario::constants::{ +use crate::planner_scenario::_helpers::values::error; +use crate::planner_scenario::_helpers::values::string; +use crate::planner_scenario::constants::{ SCENARIO_PLAN_MISSING_RELATION_TARGET, SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, }; -use crate::bindings::_helpers::planner_scenario::{ - models::SqlContext, - values::{error, string}, -}; +use crate::planner_scenario::models::SqlContext; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyStringMethods}; use sqlbuild_analysis::planner_scenario::{ @@ -20,7 +19,7 @@ use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; use sqlbuild_sqltext::sql_scan::models::Unclosed; -pub(super) fn resolve_sql( +pub(crate) fn resolve_sql( context: &SqlContext<'_>, sql: &str, project_sources: bool, @@ -41,13 +40,13 @@ pub(super) fn resolve_sql( ), (3, "dbt_ref_fixture_locations"), ] { - if project_sources && kind != super::constants::SOURCE_MARKER { + if project_sources && kind != crate::planner_scenario::constants::SOURCE_MARKER { continue; } let locations = context.relations.getattr(field)?; result = replace_markers(&result, kind, Some(syntax.clone().into()), |name, marker| { let target = mapping_get(&locations, name)?; - let target = if target.is_none() {target} else if kind == super::constants::SOURCE_MARKER { + let target = if target.is_none() {target} else if kind == crate::planner_scenario::constants::SOURCE_MARKER { object(py, render_source_relation(&target, Some(&context.adapter.clone()).filter(|value| !value.is_none()))?)? } else {native_field(&target, "qualified_name")?}; if target.is_none() { @@ -68,7 +67,7 @@ pub(super) fn resolve_sql( Ok(result) } -pub(super) fn resolve_model_dbt_refs(sql: &str, relations: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn resolve_model_dbt_refs(sql: &str, relations: &Bound<'_, PyAny>) -> PyResult { let locations = native_field(relations, "dbt_ref_fixture_locations")?; replace_markers(sql, 3, None, |name, marker| { let target = mapping_get(&locations, name)?; @@ -91,7 +90,7 @@ pub(super) fn resolve_model_dbt_refs(sql: &str, relations: &Bound<'_, PyAny>) -> }) } -pub(super) fn source_label(scenario: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn source_label(scenario: &Bound<'_, PyAny>) -> PyResult { let source = native_field(scenario, "source_path")?; if source.is_truthy()? { Ok(source.str()?.to_str()?.to_owned()) @@ -100,7 +99,7 @@ pub(super) fn source_label(scenario: &Bound<'_, PyAny>) -> PyResult { } } -pub(super) fn required_target<'py>( +pub(crate) fn required_target<'py>( relations: &Bound<'py, PyAny>, field: &str, name: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/values.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/_helpers/values.rs index b15067e5d4..9cbae543c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/values.rs @@ -6,11 +6,11 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; -pub(super) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { +pub(crate) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { PyString::new(py, text).into_any() } -pub(super) fn error( +pub(crate) fn error( py: Python<'_>, message: &str, code: &str, diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/constants.rs b/crates/sqlbuild-python-planner/src/planner_scenario/constants.rs new file mode 100644 index 0000000000..8d76b2f40c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/constants.rs @@ -0,0 +1,44 @@ +//! Public constants of planner scenario. + +pub(crate) const REF: &str = "ref"; + +pub(crate) const SOURCE: &str = "source"; + +pub(crate) const SOURCE_MARKER: usize = 2; + +pub(crate) const DBT_REF: &str = "dbt_ref"; + +pub(crate) const SCENARIO_PLAN_INVALID_HASH_PREFIX: &str = "S501"; + +pub(crate) const SCENARIO_PLAN_HASH_COLLISION: &str = "S502"; + +pub(crate) const SCENARIO_PLAN_RELATION_COLLISION: &str = "S503"; + +pub(crate) const SCENARIO_PLAN_GRAPH_VALIDATION: &str = "S504"; + +pub(crate) const SCENARIO_PLAN_UNKNOWN_SEED: &str = "S507"; + +pub(crate) const SCENARIO_PLAN_MISSING_FIXTURE_SQL: &str = "S508"; + +pub(crate) const SCENARIO_PLAN_MISSING_RELATION_TARGET: &str = "S509"; + +pub(crate) const SCENARIO_PLAN_INVALID_FIXTURE: &str = "S510"; + +pub(crate) const SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER: &str = "S511"; + +pub(crate) const SCENARIO_PLAN_EXPECTED_COLUMNS: &str = "S512"; + +pub(crate) const SCENARIO_EXPECTED_COLUMNS_HELP: &str = "list every column the model outputs in the expected CTE, and remove columns it does not output"; + +pub(crate) const SCENARIO_PLAN_INTERNAL: &str = "S599"; + +/// The zero-argument cursor intrinsics a model query can call. +pub(crate) const CURSOR_INTRINSIC_NAMES: [&str; 2] = ["__cursor_start", "__cursor_end"]; + +pub(crate) const INCREMENTAL_MODE_KEY: &str = "incremental_mode"; + +pub(crate) const MICROBATCH_INCREMENTAL_MODE: &str = "microbatch"; + +pub(crate) const CURSOR_TYPE_KEY: &str = "cursor_type"; + +pub(crate) const INTEGER_CURSOR_TYPE: &str = "integer"; diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_cli_scenario_plan_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_cli_scenario_plan_function.rs new file mode 100644 index 0000000000..6693c30336 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_cli_scenario_plan_function.rs @@ -0,0 +1,10 @@ +//! `build_cli_scenario_plan` as a function of `module`. + +use crate::planner_scenario::_helpers::cli::build_cli_scenario_plan; + +/// `build_cli_scenario_plan` as a function of `module`. +pub fn build_cli_scenario_plan_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_cli_scenario_plan, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_artifact_name_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_artifact_name_function.rs new file mode 100644 index 0000000000..539103a224 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_artifact_name_function.rs @@ -0,0 +1,10 @@ +//! `build_scenario_artifact_name` as a function of `module`. + +use crate::planner_scenario::_helpers::artifacts::build_scenario_artifact_name; + +/// `build_scenario_artifact_name` as a function of `module`. +pub fn build_scenario_artifact_name_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_scenario_artifact_name, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_execution_plan_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_execution_plan_function.rs new file mode 100644 index 0000000000..2d4b5e7ab6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_execution_plan_function.rs @@ -0,0 +1,10 @@ +//! `build_scenario_execution_plan` as a function of `module`. + +use crate::planner_scenario::_helpers::execution::build_scenario_execution_plan; + +/// `build_scenario_execution_plan` as a function of `module`. +pub fn build_scenario_execution_plan_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_scenario_execution_plan, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_fixture_plans_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_fixture_plans_function.rs new file mode 100644 index 0000000000..251599f757 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_fixture_plans_function.rs @@ -0,0 +1,10 @@ +//! `build_scenario_fixture_plans` as a function of `module`. + +use crate::planner_scenario::_helpers::fixtures::build_scenario_fixture_plans; + +/// `build_scenario_fixture_plans` as a function of `module`. +pub fn build_scenario_fixture_plans_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_scenario_fixture_plans, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_hash_index_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_hash_index_function.rs new file mode 100644 index 0000000000..cd07e2c3a8 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_hash_index_function.rs @@ -0,0 +1,10 @@ +//! `build_scenario_hash_index` as a function of `module`. + +use crate::planner_scenario::_helpers::artifacts::build_scenario_hash_index; + +/// `build_scenario_hash_index` as a function of `module`. +pub fn build_scenario_hash_index_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_scenario_hash_index, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_map_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_map_function.rs new file mode 100644 index 0000000000..4407ecf69e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_map_function.rs @@ -0,0 +1,10 @@ +//! `build_scenario_relation_map` as a function of `module`. + +use crate::planner_scenario::_helpers::artifacts::build_scenario_relation_map; + +/// `build_scenario_relation_map` as a function of `module`. +pub fn build_scenario_relation_map_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_scenario_relation_map, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_plan_of_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_plan_of_function.rs new file mode 100644 index 0000000000..463a332098 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/build_scenario_relation_plan_of_function.rs @@ -0,0 +1,10 @@ +//! `build_scenario_relation_plan_of` as a function of `module`. + +use crate::planner_scenario::_helpers::relations::build_scenario_relation_plan_of; + +/// `build_scenario_relation_plan_of` as a function of `module`. +pub fn build_scenario_relation_plan_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_scenario_relation_plan_of, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/compute_scenario_hash_prefix_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/compute_scenario_hash_prefix_function.rs new file mode 100644 index 0000000000..559bfcf7ee --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/compute_scenario_hash_prefix_function.rs @@ -0,0 +1,10 @@ +//! `compute_scenario_hash_prefix` as a function of `module`. + +use crate::planner_scenario::_helpers::artifacts::compute_scenario_hash_prefix; + +/// `compute_scenario_hash_prefix` as a function of `module`. +pub fn compute_scenario_hash_prefix_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(compute_scenario_hash_prefix, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/is_scenario_artifact_physical_name.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/is_scenario_artifact_physical_name.rs new file mode 100644 index 0000000000..6ad7d5896c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/is_scenario_artifact_physical_name.rs @@ -0,0 +1,9 @@ +//! Is scenario artifact physical name. + +use pyo3::prelude::pyfunction; +use sqlbuild_analysis::planner_scenario::main::parse_artifact_name; + +#[pyfunction] +pub fn is_scenario_artifact_physical_name(name: &str) -> bool { + parse_artifact_name::parse_artifact_name(name).is_some() +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/mod.rs new file mode 100644 index 0000000000..646c568a99 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/mod.rs @@ -0,0 +1,14 @@ +pub mod build_cli_scenario_plan_function; +pub mod build_scenario_artifact_name_function; +pub mod build_scenario_execution_plan_function; +pub mod build_scenario_fixture_plans_function; +pub mod build_scenario_hash_index_function; +pub mod build_scenario_relation_map_function; +pub mod build_scenario_relation_plan_of_function; +pub mod compute_scenario_hash_prefix_function; +pub mod is_scenario_artifact_physical_name; +pub mod parse_scenario_artifact_physical_name_function; +pub mod plan_scenario_graph; +pub mod plan_scenario_graph_function; +pub mod resolve_scenario_check_sql_function; +pub mod scenario_artifact_physical_name_check_function; diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/parse_scenario_artifact_physical_name_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/parse_scenario_artifact_physical_name_function.rs new file mode 100644 index 0000000000..97c7b22b55 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/parse_scenario_artifact_physical_name_function.rs @@ -0,0 +1,10 @@ +//! `parse_scenario_artifact_physical_name` as a function of `module`. + +use crate::planner_scenario::_helpers::artifacts::parse_scenario_artifact_physical_name; + +/// `parse_scenario_artifact_physical_name` as a function of `module`. +pub fn parse_scenario_artifact_physical_name_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(parse_scenario_artifact_physical_name, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph.rs new file mode 100644 index 0000000000..cc0a8feb14 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph.rs @@ -0,0 +1,21 @@ +//! Plan scenario graph. + +use crate::planner_scenario::_helpers::graph; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyTuple; + +/// Plan scenario graph. +#[inline] +pub fn plan_scenario_graph<'py>( + scenario: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, + sql_analysis_enabled: bool, + sql_analysis_dialect: Option<&str>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyTuple>)> { + graph::plan_scenario_graph( + scenario, + project, + sql_analysis_enabled, + sql_analysis_dialect, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph_function.rs new file mode 100644 index 0000000000..1d4687a6b6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/plan_scenario_graph_function.rs @@ -0,0 +1,10 @@ +//! `plan_scenario_graph` as a function of `module`. + +use crate::planner_scenario::_helpers::graph::plan_scenario_graph; + +/// `plan_scenario_graph` as a function of `module`. +pub fn plan_scenario_graph_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(plan_scenario_graph, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/resolve_scenario_check_sql_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/resolve_scenario_check_sql_function.rs new file mode 100644 index 0000000000..5ad1ba6296 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/resolve_scenario_check_sql_function.rs @@ -0,0 +1,10 @@ +//! `resolve_scenario_check_sql` as a function of `module`. + +use crate::planner_scenario::_helpers::sql::resolve_scenario_check_sql; + +/// `resolve_scenario_check_sql` as a function of `module`. +pub fn resolve_scenario_check_sql_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(resolve_scenario_check_sql, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/main/scenario_artifact_physical_name_check_function.rs b/crates/sqlbuild-python-planner/src/planner_scenario/main/scenario_artifact_physical_name_check_function.rs new file mode 100644 index 0000000000..1a76d55c8c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/main/scenario_artifact_physical_name_check_function.rs @@ -0,0 +1,10 @@ +//! `is_scenario_artifact_physical_name` as a function of `module`. + +use crate::planner_scenario::main::is_scenario_artifact_physical_name::is_scenario_artifact_physical_name; + +/// `is_scenario_artifact_physical_name` as a function of `module`. +pub fn scenario_artifact_physical_name_check_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(is_scenario_artifact_physical_name, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/mod.rs b/crates/sqlbuild-python-planner/src/planner_scenario/mod.rs new file mode 100644 index 0000000000..86c8fdf294 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_scenario/mod.rs @@ -0,0 +1,6 @@ +//! Python boundaries for native scenario planning. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs b/crates/sqlbuild-python-planner/src/planner_scenario/models.rs similarity index 52% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs rename to crates/sqlbuild-python-planner/src/planner_scenario/models.rs index 70ff75b823..b385cc1b2f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/models.rs @@ -1,4 +1,4 @@ -//! Borrowed Python callback boundaries for scenario SQL assembly. +//! Public records of planner scenario. use pyo3::prelude::{Bound, PyAny}; @@ -13,10 +13,10 @@ pub(crate) struct SqlContext<'py> { /// One scenario's graph and the relation map and qualification that locate its relations. #[derive(Debug)] pub(crate) struct ScenarioRelations<'py> { - pub(crate) project: Bound<'py, PyAny>, - pub(crate) graph_plan: Bound<'py, PyAny>, - pub(crate) relation_map: Bound<'py, PyAny>, - pub(crate) render_qualified_name: Bound<'py, PyAny>, - pub(crate) database: Bound<'py, PyAny>, - pub(crate) schema: Bound<'py, PyAny>, + pub project: Bound<'py, PyAny>, + pub graph_plan: Bound<'py, PyAny>, + pub relation_map: Bound<'py, PyAny>, + pub render_qualified_name: Bound<'py, PyAny>, + pub database: Bound<'py, PyAny>, + pub schema: Bound<'py, PyAny>, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs similarity index 76% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs index 6617268137..f94e6c0176 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs @@ -1,31 +1,25 @@ //! Assemble audit plan dataclasses using native rendering and scheduling. -use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::models::GraphIndexes; -use std::collections::HashSet; -use crate::bindings::_helpers::planner_strategy::audits::{ +use crate::planner_strategy::_helpers::audits::{ attachment, outside_target_lineage, resolve_effective_run_scope, }; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PyString}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_front::attachments::main::assert_no_unresolved_sql_markers::assert_no_unresolved_sql_markers; use sqlbuild_python_project::auditing::main::render_audit_sql::render_audit_sql; -use sqlbuild_python_project::compiled_views::main::project::audits::audits; -use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::graph::models::NativeProjectGraph; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_scope::main::edges::edges; -use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use sqlbuild_python_project::resource_views::main::audit_reads::always_run::always_run; use sqlbuild_python_project::resource_views::main::audit_reads::attached_column_name::attached_column_name; use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_kind::attached_target_kind; -use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_name::attached_target_name; use sqlbuild_python_project::resource_views::main::audit_reads::definition_name::definition_name; use sqlbuild_python_project::resource_views::main::audit_reads::description::description; use sqlbuild_python_project::resource_views::main::audit_reads::evaluation_mode::evaluation_mode; @@ -122,7 +116,7 @@ pub(crate) fn plan_audit_of<'py>( } /// One audit's plan entry, rendered against the planning relations. -fn plan_audit<'py>( +pub(crate) fn plan_audit<'py>( audit: &Bound<'py, PyAny>, request: &AuditPlanning<'py>, ) -> PyResult> { @@ -228,61 +222,3 @@ fn plan_audit<'py>( } new_record(py, RecordId::AuditPlanEntry, (), Some(&values)) } - -/// Select attached audits by their logical target and singular audits by dependency overlap. -pub(crate) fn audit_is_selected( - audit: &Bound<'_, PyAny>, - selected_keys: &HashSet, -) -> PyResult { - let py = audit.py(); - let kind = attached_target_kind(audit)?; - let name = attached_target_name(audit)?; - if !kind.is_none() && !name.is_none() { - let target_value = native_field( - &imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind,))?, - "value", - )?; - let key = (target_value.extract()?, name.extract()?); - return Ok(selected_keys.contains(&key)); - } - for dependency in scope_deps(audit)?.try_iter()? { - if selected_keys.contains(&native_key(&dependency?)?) { - return Ok(true); - } - } - Ok(false) -} - -/// Assemble selected audits in compiled-project order without querying lineage for an empty selection. -pub(crate) fn build_selected_audit_entries<'py>( - request: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let request = Request::new("build_selected_audit_entries", request)?; - let py = request.py(); - let project = request.get("project")?; - let scope = scope_data(&request.get("scope")?)?; - let mut selected: Vec> = Vec::new(); - for audit in audits(&project)?.try_iter()? { - let audit = audit?; - if audit_is_selected(&audit, &scope.selected)? { - selected.push(audit); - } - } - let entries = PyList::empty(py); - if selected.is_empty() { - return Ok(entries); - } - let relations = request.get("relations")?; - let planning = AuditPlanning { - model_locations: native_field(&relations, "model_locations")?, - seed_locations: native_field(&relations, "seed_locations")?, - source_map: native_field(&relations, "source_read_map")?, - adapter: request.get("adapter")?, - model_materializations: request.get("model_materializations")?, - lineage: native_field(&project, "lineage_graph")?.extract()?, - }; - for audit in selected { - entries.append(plan_audit(&audit, &planning)?)?; - } - Ok(entries) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs index 3c4e27c50a..d393d68ca8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs @@ -1,11 +1,13 @@ //! Audit attachment, target lineage and run scope. +use crate::planner_strategy::main::values::input_error::input_error; +use crate::planner_strategy::main::values::text::text; use std::cell::RefCell; use std::collections::HashMap; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::main::reachable_graph_keys::reachable_graph_keys; @@ -15,7 +17,7 @@ use sqlbuild_analysis::planner_strategy::main::effective_run_scope::effective_ru use sqlbuild_analysis::planner_strategy::main::singular_attachment::singular_attachment; use sqlbuild_analysis::planner_strategy::models::{AuditAttachmentKind, AuditRunScope}; -use crate::bindings::_helpers::planner_strategy::values::{input_error, optional_string, text}; +use crate::planner_strategy::_helpers::values::optional_string; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; @@ -185,9 +187,3 @@ pub(crate) fn resolve_effective_run_scope<'py>( }; imported(py, AUDITING_TYPES_MODULE, "AuditRunScope")?.call1((scope,)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_attachment_kind, module)?)?; - module.add_function(wrap_pyfunction!(reads_outside_target_lineage, module)?)?; - module.add_function(wrap_pyfunction!(resolve_effective_run_scope, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_inputs.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_inputs.rs index 37b06bfb0a..24b958f4ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_inputs.rs @@ -1,5 +1,8 @@ //! Cursor input relations and authored cursor input column checks of model plan entries. +use crate::planner_strategy::main::lineage::resolve_lineage_reference::resolve_lineage_reference; +use crate::planner_strategy::main::values::input_error::input_error; +use crate::planner_strategy::main::values::text::text; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyFrozenSet, PyString, PyTuple}; @@ -7,9 +10,7 @@ use pyo3::types::{PyBool, PyDict, PyFrozenSet, PyString, PyTuple}; use sqlbuild_analysis::planner_strategy::main::cursor_input_column::cursor_input_column_failure; use sqlbuild_analysis::planner_strategy::models::{CursorInputColumnCheck, CursorInputColumnOwner}; -use crate::bindings::_helpers::planner_identity::metadata::or_empty; -use crate::bindings::_helpers::planner_strategy::lineage::resolve_lineage_reference; -use crate::bindings::_helpers::planner_strategy::values::{input_error, text}; +use crate::planner_identity::main::or_empty::or_empty; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -24,7 +25,6 @@ use sqlbuild_python_front::model_config::main::config::config_text_object::confi use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; -use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use sqlbuild_python_project::planner_strategy::constants::{ ENFORCED_CONTRACT, INCREMENTAL_MATERIALIZATION, MODEL_REFERENCE, SEED_REFERENCE, SOURCE_REFERENCE, TIMESTAMP_CURSOR, @@ -47,7 +47,7 @@ fn incremental_cursor_flag( } /// The names of a sequence of objects with a `name`. -fn names(items: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn names(items: &Bound<'_, PyAny>) -> PyResult> { items .try_iter()? .map(|item| text(&native_field(&item?, "name")?)) @@ -333,28 +333,3 @@ pub(crate) fn build_cursor_input_relations<'py>( } PyTuple::new(py, relations) } - -/// Selected models and seeds, and selected sources with loaders, whose relations this invocation produces. -#[pyfunction] -#[pyo3(signature = (*, scope, source_map))] -pub(crate) fn runtime_cursor_producer_names<'py>( - scope: &Bound<'py, PyAny>, - source_map: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = scope.py(); - let data = scope_data(scope)?; - let mut names: Vec<&str> = Vec::new(); - for (kind, name) in &data.selected { - match kind.as_str() { - "model" | "seed" => names.push(name), - "source" => { - let source: Bound<'py, PyAny> = mapping_get(source_map, name)?; - if !source.is_none() && !native_field(&source, "loader")?.is_none() { - names.push(name); - } - } - _ => {} - } - } - PyFrozenSet::new(py, names) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs similarity index 68% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs index 126310f583..3fa759c175 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs @@ -1,28 +1,26 @@ //! Warehouse cursor column type checks. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::planner_strategy::main::values::text::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_strategy::errors::TypeParseFailure; -use sqlbuild_analysis::planner_strategy::main::classify_cursor_sql_type::classify_cursor_sql_type as classify; use sqlbuild_analysis::planner_strategy::main::cursor_type_warning::cursor_type_warning; -use sqlbuild_analysis::planner_strategy::models::{CursorType, CursorTypeCheck}; +use sqlbuild_analysis::planner_strategy::models::CursorTypeCheck; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::planner_changes::results::planner_enum; -use crate::bindings::_helpers::planner_strategy::values::{optional_string, plan_warning, text}; +use crate::planner_strategy::_helpers::values::{optional_string, plan_warning}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; const PLANNER_LOGGER: &str = "sqlbuild.planner"; /// Log the Polyglot parse error that left a warehouse type unclassified. -fn log_parse_failure(py: Python<'_>, failure: &TypeParseFailure) -> PyResult<()> { +pub(crate) fn log_parse_failure(py: Python<'_>, failure: &TypeParseFailure) -> PyResult<()> { let text = |value: &str| PyString::new(py, value).into_any(); call_with( &host_entry(py, ExecutorHost::LogDebugEvent)?, @@ -42,33 +40,10 @@ fn log_parse_failure(py: Python<'_>, failure: &TypeParseFailure) -> PyResult<()> .map(|_| ()) } -/// The `CursorType` a supported cross-adapter SQL type holds, or `None`. -pub(crate) fn classify_cursor_sql_type<'py>( - py: Python<'py>, - sql_type: &str, -) -> PyResult>> { - let detected: Option = match classify(sql_type) { - Ok(detected) => detected, - Err(failure) => { - log_parse_failure(py, &failure)?; - None - } - }; - detected - .map(|cursor_type| { - let value: &str = match cursor_type { - CursorType::Timestamp => "timestamp", - CursorType::Integer => "integer", - }; - planner_enum(py, RecordId::CursorType, value) - }) - .transpose() -} - /// A `PlanWarning` when the warehouse cursor column type contradicts `cursor_type`. #[pyfunction] #[pyo3(signature = (**options))] -fn check_cursor_type_consistency<'py>( +pub(crate) fn check_cursor_type_consistency<'py>( py: Python<'py>, options: Option>, ) -> PyResult>> { @@ -124,7 +99,3 @@ pub(crate) fn cursor_type_warning_record<'py>( } warning.map(|row| plan_warning(model_name, row)).transpose() } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(check_cursor_type_consistency, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs similarity index 60% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs index 40fdaa4647..d81b0868fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs @@ -1,18 +1,14 @@ //! Watermark inputs resolved through SQL reference lineage, and capped producers behind them. +use crate::planner_strategy::main::values::input_error::input_error; +use crate::planner_strategy::main::values::text::text; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::PyDict; use sqlbuild_analysis::planner_strategy::errors::LineageFailure; -use sqlbuild_analysis::planner_strategy::main::capped_watermark_input::capped_watermark_input; -use sqlbuild_analysis::planner_strategy::main::lineage_reference::lineage_reference; use sqlbuild_analysis::planner_strategy::models::{ProducerFacts, ReferenceIdentity}; use sqlbuild_analysis::planner_strategy::types::ReferenceGraph; -use crate::bindings::_helpers::planner_strategy::microbatches::{ - CAP_ACTIONS, microbatch_limit_config, -}; -use crate::bindings::_helpers::planner_strategy::values::{input_error, text}; +use crate::planner_strategy::_helpers::microbatches::{CAP_ACTIONS, microbatch_limit_config}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; @@ -20,15 +16,14 @@ use sqlbuild_python_front::model_config::main::config::config_text_is::config_te use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; use sqlbuild_python_project::compiled_views::main::compiled_models::config::config; -use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; use sqlbuild_python_project::native_host::models::ModelLookup; const MICROBATCH_MODE: &str = "microbatch"; /// The project's models and SQL functions by name. -struct ProjectLineage<'py> { - models_by_name: ModelLookup, - functions_by_name: Bound<'py, PyAny>, +pub(crate) struct ProjectLineage<'py> { + pub(crate) models_by_name: ModelLookup, + pub(crate) functions_by_name: Bound<'py, PyAny>, } /// The references of the value under `name`, when the mapping has it. @@ -43,14 +38,14 @@ fn mapped_references<'py>( references(&value).map(Some) } -fn references<'py>(owner: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn references<'py>(owner: &Bound<'py, PyAny>) -> PyResult>> { sqlbuild_python_project::resource_views::main::resource_reads::references::references(owner)? .try_iter()? .collect() } /// Whether a model's `incremental_mode` is microbatch. -fn microbatch_flag(model: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn microbatch_flag(model: &Bound<'_, PyAny>) -> PyResult { config_text_is( model.py(), &model_values(model)?, @@ -60,7 +55,7 @@ fn microbatch_flag(model: &Bound<'_, PyAny>) -> PyResult { } /// A model's watermark input names. -fn watermark_names(model: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn watermark_names(model: &Bound<'_, PyAny>) -> PyResult> { native_field(&resolve_cursor_input_roles(model)?, "watermark_inputs")? .try_iter()? .map(|name| text(&name?)) @@ -109,7 +104,7 @@ impl<'py> ReferenceGraph for ProjectLineage<'py> { } /// The Python error of a failed lineage walk. -fn lineage_error(py: Python<'_>, failure: LineageFailure) -> PyErr { +pub(crate) fn lineage_error(py: Python<'_>, failure: LineageFailure) -> PyErr { match failure { LineageFailure::Graph(error) => error, LineageFailure::NotInLineage { model, input } => input_error( @@ -120,52 +115,3 @@ fn lineage_error(py: Python<'_>, failure: LineageFailure) -> PyErr { ), } } - -/// The reference one named input resolves to through model and SQL function references. -pub(crate) fn resolve_lineage_reference<'py>( - model: &Bound<'py, PyAny>, - input_name: &str, - models_by_name: ModelLookup, - functions_by_name: Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = model.py(); - let graph = ProjectLineage { - models_by_name, - functions_by_name, - }; - let name: String = model_name(model)?; - lineage_reference(&graph, (&name, references(model)?), input_name) - .map_err(|failure| lineage_error(py, failure)) -} - -/// Reject capped producers feeding a microbatch model's watermark inputs. -pub(crate) fn reject_capped_watermark_inputs<'py>( - model: &Bound<'py, PyAny>, - models_by_name: ModelLookup, - functions_by_name: Option>, -) -> PyResult<()> { - let py: Python<'py> = model.py(); - if !microbatch_flag(model)? { - return Ok(()); - } - let functions_by_name: Bound<'py, PyAny> = match functions_by_name { - Some(functions) if functions.is_truthy()? => functions, - _ => PyDict::new(py).into_any(), - }; - let graph = ProjectLineage { - models_by_name, - functions_by_name, - }; - let name: String = model_name(model)?; - let inputs: Vec = watermark_names(model)?; - match capped_watermark_input(&graph, (&name, references(model)?), inputs) { - Ok(None) => Ok(()), - Ok(Some(capped)) => Err(input_error( - py, - ErrorId::CompileInput, - &capped.message(), - None, - )), - Err(failure) => Err(lineage_error(py, failure)), - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs similarity index 79% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs index 9153633367..a51ea10ee2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs @@ -1,10 +1,11 @@ //! Microbatch limits applied to planned model entries in CLI, project and model precedence. -use crate::bindings::_helpers::planner_changes::results::native_enum; +use crate::planner_changes::main::native_enum::native_enum; +use crate::planner_strategy::main::values::input_error::input_error; +use crate::planner_strategy::main::values::text::text; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction}; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; @@ -14,11 +15,11 @@ use sqlbuild_core::plan_display::errors::UnknownCursorGrain; use sqlbuild_core::plan_display::main::effective_timestamp_grain::effective_timestamp_grain; use sqlbuild_core::plan_display::models::TimestampGrainFacts; -use crate::bindings::_helpers::plan_output::display::resolve_effective_microbatch_batch_size; -use crate::bindings::_helpers::planner_strategy::microbatches::{ +use crate::plan_output::main::resolve_effective_microbatch_batch_size::resolve_effective_microbatch_batch_size; +use crate::planner_strategy::_helpers::microbatches::{ CAP_ACTIONS, capped_microbatch_bounds, count_microbatches, }; -use crate::bindings::_helpers::planner_strategy::values::{input_error, optional_string, text}; +use crate::planner_strategy::_helpers::values::optional_string; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; @@ -91,7 +92,7 @@ fn batch_count(entry: &Bound<'_, PyAny>) -> PyResult> { } /// The entry with one batch limit applied, and the warning when a non-error limit is exceeded. -fn limited<'py>( +pub(crate) fn limited<'py>( entry: &Bound<'py, PyAny>, max_batches: Bound<'py, PyAny>, action: Bound<'py, PyAny>, @@ -192,7 +193,7 @@ pub(crate) fn apply_microbatch_limit<'py>( } /// `(entry, warnings)` with the limit outcome's warning, if any. -fn with_warnings<'py>( +pub(crate) fn with_warnings<'py>( (entry, warning): LimitedEntry<'py>, mut warnings: Vec>, ) -> PyResult> { @@ -202,7 +203,7 @@ fn with_warnings<'py>( } /// The default limit: none for an incremental rolling window, else the model's limit or the configured ceiling. -fn default_limit<'py>( +pub(crate) fn default_limit<'py>( entry: &Bound<'py, PyAny>, configured: Bound<'py, PyAny>, ) -> PyResult> { @@ -221,39 +222,3 @@ fn default_limit<'py>( configured }) } - -/// Apply CLI, project, and model batch limits in their documented precedence order. -#[pyfunction] -#[pyo3(signature = (*, entry, inputs))] -pub(crate) fn apply_configured_microbatch_limits<'py>( - entry: &Bound<'py, PyAny>, - inputs: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = entry.py(); - let configured: Bound<'py, PyAny> = native_field(inputs, "max_microbatches")?; - let configured_action: Bound<'py, PyAny> = native_field(inputs, "microbatch_limit_action")?; - if native_field(inputs, "max_microbatches_is_override")?.is_truthy()? { - let error: Bound<'py, PyAny> = enum_member(py, RecordId::MicrobatchLimitAction, "ERROR")?; - return with_warnings(limited(entry, configured, error)?, Vec::new()); - } - let declared: Bound<'py, PyAny> = native_field(entry, "declared_microbatch_limit_action")?; - if declared.is_none() { - let maximum: Bound<'py, PyAny> = default_limit(entry, configured)?; - return with_warnings(limited(entry, maximum, configured_action)?, Vec::new()); - } - let mut warnings: Vec> = Vec::new(); - let mut entry: Bound<'py, PyAny> = entry.clone(); - if !configured.is_none() { - let (_, safety_warning) = limited(&entry, configured.clone(), configured_action.clone())?; - entry = replace( - &entry, - &[ - ("microbatch_safety_limit", configured), - ("microbatch_safety_limit_action", configured_action), - ], - )?; - warnings.extend(safety_warning); - } - let model_limit: Bound<'py, PyAny> = native_field(&entry, "microbatch_limit")?; - with_warnings(limited(&entry, model_limit, declared)?, warnings) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatches.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatches.rs index 239ab716a3..3a5482af5b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatches.rs @@ -1,12 +1,11 @@ //! Microbatch counts, capped ranges and batch-limit configuration. +use crate::planner_strategy::main::values::text::text; use std::marker::PhantomData; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyString, PyTuple, PyTupleMethods}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::planner_strategy::main::cap_integer_bounds::cap_integer_bounds; use sqlbuild_analysis::planner_strategy::main::cap_timeline_bounds::cap_timeline_bounds; @@ -18,8 +17,7 @@ use sqlbuild_analysis::planner_strategy::types::Timeline; use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; use sqlbuild_cursor::algebra::models::{DurationDigits, DurationSpan}; -use crate::bindings::_helpers::plan_output::display::aligned_timestamp_bounds; -use crate::bindings::_helpers::planner_strategy::values::text; +use crate::plan_output::main::aligned_timestamp_bounds::aligned_timestamp_bounds; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; @@ -236,7 +234,7 @@ pub(crate) fn capped_microbatch_bounds<'py>( /// The prominent warning or error message when one model exceeds its batch limit. #[pyfunction] #[pyo3(signature = (*, model_name, max_batches, batch_count, action))] -fn microbatch_limit_warning( +pub(crate) fn microbatch_limit_warning( model_name: &str, max_batches: Option, batch_count: i128, @@ -283,8 +281,3 @@ pub(crate) fn microbatch_limit_config<'py>( let legacy: Bound<'py, PyAny> = mapping_get(values, "max_microbatches")?; Ok((positive_integer(Some(legacy))?, None)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(microbatch_limit_warning, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/mod.rs new file mode 100644 index 0000000000..e1e98c41e6 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod audit_entries; +pub(crate) mod audits; +pub(crate) mod cursor_inputs; +pub(crate) mod cursor_types; +pub(crate) mod lineage; +pub(crate) mod microbatch_limits; +pub(crate) mod microbatches; +pub(crate) mod model_entries; +pub(crate) mod strategy; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs index 6d08220ae0..bc5e45b43f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs @@ -1,9 +1,11 @@ //! Assemble model plan dataclasses from resolved plan decisions and authored model config. +use crate::planner_strategy::main::values::input_error::input_error; +use crate::planner_strategy::main::values::text::text; +use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::planner_strategy::main::group_replacement_refusal::group_replacement_refusal; use sqlbuild_analysis::planner_strategy::main::materialization_type::materialization_type; @@ -17,17 +19,15 @@ use sqlbuild_model_config::execution_limits::main::parse_execution_limit::parse_ use sqlbuild_model_config::execution_limits::models::ExecutionLimit; use sqlbuild_model_config::model_validation::constants::PHYSICAL_STORAGE_KEYS; -use crate::bindings::_helpers::planner_strategy::cursor_types::{ +use crate::planner_strategy::_helpers::cursor_types::{ cursor_type_warning_record, warehouse_column_types, }; -use crate::bindings::_helpers::planner_strategy::microbatch_limits::{replay_grain, unknown_grain}; -use crate::bindings::_helpers::planner_strategy::microbatches::microbatch_limit_config; -use crate::bindings::_helpers::planner_strategy::strategy::{ +use crate::planner_strategy::_helpers::microbatch_limits::{replay_grain, unknown_grain}; +use crate::planner_strategy::_helpers::microbatches::microbatch_limit_config; +use crate::planner_strategy::_helpers::strategy::{ model_warning_records, resolve_schema_actions, warning_inputs, }; -use crate::bindings::_helpers::planner_strategy::values::{ - input_error, optional_string, plan_warning, text, -}; +use crate::planner_strategy::_helpers::values::{optional_string, plan_warning}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; @@ -112,7 +112,7 @@ fn column_info<'py>( } /// Declared schema columns as `ColumnInfo`: typed columns only, or every column with an empty default type. -fn schema_columns<'py>( +pub(crate) fn schema_columns<'py>( model: &Bound<'py, PyAny>, typed_only: bool, ) -> PyResult> { @@ -227,7 +227,7 @@ fn type_enforcement<'py>(model: &Bound<'py, PyAny>) -> PyResult, effective_grain: Option, ) -> PyResult> { @@ -249,21 +249,6 @@ fn lookback_of( microbatch_lookback(&config).map_err(|error| unknown_grain(model.py(), error)) } -/// Resolve explicit or default lookback for one microbatch model. -#[pyfunction] -#[pyo3(signature = (*, model, effective_grain = None))] -pub(crate) fn resolve_microbatch_lookback( - model: &Bound<'_, PyAny>, - effective_grain: Option, -) -> PyResult> { - lookback_of(model, effective_grain) -} - -/// Declared seed columns with types, as `ColumnInfo`. -pub(crate) fn extract_seed_columns<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - schema_columns(seed, true) -} - /// The adapter-rendered logical DDL of one plan action. fn logical_ddl<'py>( request: &ModelEntryDecisions<'py>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs similarity index 62% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs index d68f6cd890..fe225e30fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs @@ -1,29 +1,24 @@ //! Per-model plan actions, schema actions and plan warnings. -use crate::bindings::_helpers::planner_changes::results::native_enum; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use crate::planner_changes::main::native_enum::native_enum; +use crate::planner_strategy::main::values::text::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_analysis::planner_strategy::main::materialization_type::materialization_type; -use sqlbuild_analysis::planner_strategy::main::model_plan_action::model_plan_action; use sqlbuild_analysis::planner_strategy::main::model_warnings::model_warnings; use sqlbuild_analysis::planner_strategy::main::schema_actions::schema_actions; use sqlbuild_analysis::planner_strategy::models::{ - ModelStrategy, PlanAction, PlanReason, SchemaActionKind, WarningInputs, + PlanAction, PlanReason, SchemaActionKind, WarningInputs, }; -use crate::bindings::_helpers::planner_changes::results::planner_enum; -use crate::bindings::_helpers::planner_strategy::values::{ - change_facts, config_value, finding_rows, input_error, materialization_value, - model_materialization, optional_string, plan_warning, text, -}; +use crate::planner_strategy::_helpers::values::{change_facts, finding_rows, plan_warning}; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -fn action_value(action: PlanAction) -> &'static str { +pub(crate) fn action_value(action: PlanAction) -> &'static str { match action { PlanAction::CreateView => "create_view", PlanAction::CreateTable => "create_table", @@ -36,7 +31,7 @@ fn action_value(action: PlanAction) -> &'static str { } } -fn reason_value(reason: PlanReason) -> &'static str { +pub(crate) fn reason_value(reason: PlanReason) -> &'static str { match reason { PlanReason::FirstRun => "first_run", PlanReason::Renamed => "renamed", @@ -59,56 +54,6 @@ fn schema_policy(on_schema_change: &Bound<'_, PyAny>) -> PyResult text(on_schema_change).map(Some) } -/// The `MaterializationType` of a model's `materialized` config: table when unset, custom when unknown. -#[pyfunction(name = "get_materialization_type")] -#[pyo3(signature = (model))] -pub(crate) fn materialization_of<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - planner_enum( - model.py(), - RecordId::MaterializationType, - materialization_value(model_materialization(model)?), - ) -} - -/// The plan action and reason for one model. -#[pyfunction] -#[pyo3(signature = (*, model, change_result, full_refresh))] -pub(crate) fn resolve_model_plan_action<'py>( - model: &Bound<'py, PyAny>, - change_result: &Bound<'py, PyAny>, - full_refresh: bool, -) -> PyResult> { - let py: Python<'py> = model.py(); - let enabled: Bound<'py, PyAny> = config_value(model, "enabled")?; - let strategy = ModelStrategy { - name: - sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name( - model, - )?, - disabled: enabled.is_instance_of::() && !enabled.is_truthy()?, - materialization: model_materialization(model)?, - incremental_strategy: optional_string(&config_value(model, "incremental_strategy")?)?, - }; - let (action, reason) = - model_plan_action(&strategy, &change_facts(change_result)?, full_refresh).map_err( - |error| { - input_error( - py, - ErrorId::PlannerInput, - &error.message(), - Some(error.code()), - ) - }, - )?; - PyTuple::new( - py, - [ - planner_enum(py, RecordId::PlanAction, action_value(action))?, - planner_enum(py, RecordId::PlanReason, reason_value(reason))?, - ], - ) -} - /// Concrete schema change actions from findings and `on_schema_change`. #[pyfunction] #[pyo3(signature = (*, schema_findings, on_schema_change))] @@ -175,7 +120,7 @@ pub(crate) fn model_warning_records<'py>( /// Warnings for one model from its change detection and plan decisions. #[pyfunction] #[pyo3(signature = (**options))] -fn build_model_warnings<'py>( +pub(crate) fn build_model_warnings<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -210,10 +155,3 @@ fn build_model_warnings<'py>( model_warning_records(&model_name, &inputs, &arguments.value("change_result")?)?, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(materialization_of, module)?)?; - module.add_function(wrap_pyfunction!(resolve_model_plan_action, module)?)?; - module.add_function(wrap_pyfunction!(resolve_schema_actions, module)?)?; - module.add_function(wrap_pyfunction!(build_model_warnings, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs rename to crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs index 22e3989765..d910fd16b3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs @@ -1,9 +1,10 @@ //! Conversions between the planner's Python models and the native strategy rows. -use crate::bindings::_helpers::planner_changes::results::native_enum; +use crate::planner_changes::main::native_enum::native_enum; +use crate::planner_strategy::main::values::text::text; use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_front::model_config::main::config::config_value::config_value as typed_config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; @@ -16,20 +17,13 @@ use sqlbuild_analysis::planner_strategy::models::{ ChangeFacts, MaterializationType, PlanChangeKind, WarningRow, WarningSeverity, }; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; const YML_SOURCE: &str = "yml"; const FULL_BACKFILL: &str = "full"; const FORWARD_ONLY_BACKFILL: &str = "forward"; -/// `str(value)` of a Python string or string enum. -pub(crate) fn text(value: &Bound<'_, PyAny>) -> PyResult { - value.extract() -} - /// A string value, or `None` for `None` and anything that is not a string. pub(crate) fn optional_string(value: &Bound<'_, PyAny>) -> PyResult> { if value.is_instance_of::() { @@ -149,23 +143,3 @@ pub(crate) fn plan_warning<'py>( ], ) } - -/// The native error `class(message, code=code)`. -pub(crate) fn input_error( - py: Python<'_>, - class: ErrorId, - message: &str, - code: Option<&str>, -) -> PyErr { - let raised: PyResult> = error_class(py, class).and_then(|class| { - let options: Bound<'_, PyDict> = PyDict::new(py); - if let Some(code) = code { - options.set_item("code", code)?; - } - class.call((message,), Some(&options)) - }); - match raised { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } -} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs new file mode 100644 index 0000000000..ab7a81e959 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs @@ -0,0 +1,36 @@ +//! Select attached audits by their logical target and singular audits by dependency overlap. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::planner_strategy::constants::COMPILE_TYPES_MODULE; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_kind::attached_target_kind; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_name::attached_target_name; +use sqlbuild_python_project::resource_views::main::audit_reads::scope_deps::scope_deps; +use std::collections::HashSet; + +/// Select attached audits by their logical target and singular audits by dependency overlap. +pub fn audit_is_selected( + audit: &Bound<'_, PyAny>, + selected_keys: &HashSet, +) -> PyResult { + let py = audit.py(); + let kind = attached_target_kind(audit)?; + let name = attached_target_name(audit)?; + if !kind.is_none() && !name.is_none() { + let target_value = native_field( + &imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind,))?, + "value", + )?; + let key = (target_value.extract()?, name.extract()?); + return Ok(selected_keys.contains(&key)); + } + for dependency in scope_deps(audit)?.try_iter()? { + if selected_keys.contains(&native_key(&dependency?)?) { + return Ok(true); + } + } + Ok(false) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs new file mode 100644 index 0000000000..82e778151a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs @@ -0,0 +1,45 @@ +//! Assemble selected audits in compiled-project order without querying lineage for an empty selection. + +use crate::planner_strategy::_helpers::audit_entries::plan_audit; +use crate::planner_strategy::main::audit_entries::audit_is_selected::audit_is_selected; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyList, PyListMethods}; +use sqlbuild_python_base::boundary::models::Request; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::project::audits::audits; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_strategy::models::AuditPlanning; + +/// Assemble selected audits in compiled-project order without querying lineage for an empty selection. +pub fn build_selected_audit_entries<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("build_selected_audit_entries", request)?; + let py = request.py(); + let project = request.get("project")?; + let scope = scope_data(&request.get("scope")?)?; + let mut selected: Vec> = Vec::new(); + for audit in audits(&project)?.try_iter()? { + let audit = audit?; + if audit_is_selected(&audit, &scope.selected)? { + selected.push(audit); + } + } + let entries = PyList::empty(py); + if selected.is_empty() { + return Ok(entries); + } + let relations = request.get("relations")?; + let planning = AuditPlanning { + model_locations: native_field(&relations, "model_locations")?, + seed_locations: native_field(&relations, "seed_locations")?, + source_map: native_field(&relations, "source_read_map")?, + adapter: request.get("adapter")?, + model_materializations: request.get("model_materializations")?, + lineage: native_field(&project, "lineage_graph")?.extract()?, + }; + for audit in selected { + entries.append(plan_audit(&audit, &planning)?)?; + } + Ok(entries) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/mod.rs new file mode 100644 index 0000000000..31bd8b5c4c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/mod.rs @@ -0,0 +1,3 @@ +pub mod audit_is_selected; +pub mod build_selected_audit_entries; +pub mod plan_audit_of_function; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/plan_audit_of_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/plan_audit_of_function.rs new file mode 100644 index 0000000000..b09754ca80 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/plan_audit_of_function.rs @@ -0,0 +1,10 @@ +//! `plan_audit_of` as a function of `module`. + +use crate::planner_strategy::_helpers::audit_entries::plan_audit_of; + +/// `plan_audit_of` as a function of `module`. +pub fn plan_audit_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(plan_audit_of, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/mod.rs new file mode 100644 index 0000000000..00e5e61db2 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/mod.rs @@ -0,0 +1 @@ +pub mod register_audits; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs new file mode 100644 index 0000000000..f1c8160e55 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs @@ -0,0 +1,14 @@ +//! Register the audits functions and classes. + +use crate::planner_strategy::_helpers::audits::reads_outside_target_lineage; +use crate::planner_strategy::_helpers::audits::resolve_attachment_kind; +use crate::planner_strategy::_helpers::audits::resolve_effective_run_scope; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the audits functions and classes. +pub fn register_audits(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_attachment_kind, module)?)?; + module.add_function(wrap_pyfunction!(reads_outside_target_lineage, module)?)?; + module.add_function(wrap_pyfunction!(resolve_effective_run_scope, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/build_cursor_input_relations.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/build_cursor_input_relations.rs new file mode 100644 index 0000000000..ea671898f1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/build_cursor_input_relations.rs @@ -0,0 +1,16 @@ +//! Cursor-bearing input relations of an incremental model, for runtime range discovery. + +use crate::planner_strategy::_helpers::cursor_inputs; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyTuple; +use sqlbuild_python_project::planner_strategy::models::CursorInputLookups; + +/// Cursor-bearing input relations of an incremental model, for runtime range discovery. +#[inline] +pub fn build_cursor_input_relations<'py>( + (model, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + cursor_column: &Bound<'py, PyAny>, + lookups: &CursorInputLookups<'py>, +) -> PyResult> { + cursor_inputs::build_cursor_input_relations((model, adapter), cursor_column, lookups) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_columns_of_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_columns_of_function.rs new file mode 100644 index 0000000000..ec1cf594b8 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_columns_of_function.rs @@ -0,0 +1,10 @@ +//! `cursor_input_columns_of` as a function of `module`. + +use crate::planner_strategy::_helpers::cursor_inputs::cursor_input_columns_of; + +/// `cursor_input_columns_of` as a function of `module`. +pub fn cursor_input_columns_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(cursor_input_columns_of, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_relations_of_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_relations_of_function.rs new file mode 100644 index 0000000000..22a132793f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/cursor_input_relations_of_function.rs @@ -0,0 +1,10 @@ +//! `cursor_input_relations_of` as a function of `module`. + +use crate::planner_strategy::_helpers::cursor_inputs::cursor_input_relations_of; + +/// `cursor_input_relations_of` as a function of `module`. +pub fn cursor_input_relations_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(cursor_input_relations_of, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/mod.rs new file mode 100644 index 0000000000..1106186cea --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/mod.rs @@ -0,0 +1,6 @@ +pub mod build_cursor_input_relations; +pub mod cursor_input_columns_of_function; +pub mod cursor_input_relations_of_function; +pub mod reject_unexposed_cursor_input_columns; +pub mod runtime_cursor_producer_names; +pub mod runtime_cursor_producer_names_function; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/reject_unexposed_cursor_input_columns.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/reject_unexposed_cursor_input_columns.rs new file mode 100644 index 0000000000..efb905a3f5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/reject_unexposed_cursor_input_columns.rs @@ -0,0 +1,15 @@ +//! Reject authored cursor input columns a contract or the source warehouse metadata does not expose. + +use crate::planner_strategy::_helpers::cursor_inputs; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_project::planner_strategy::models::CursorInputLookups; + +/// Reject authored cursor input columns a contract or the source warehouse metadata does not expose. +#[inline] +pub fn reject_unexposed_cursor_input_columns<'py>( + model: &Bound<'py, PyAny>, + cursor_column: &Bound<'py, PyAny>, + lookups: &CursorInputLookups<'py>, +) -> PyResult<()> { + cursor_inputs::reject_unexposed_cursor_input_columns(model, cursor_column, lookups) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names.rs new file mode 100644 index 0000000000..306e46dcfa --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names.rs @@ -0,0 +1,33 @@ +//! Selected models and seeds, and selected sources with loaders, whose relations this invocation produces. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyFrozenSet; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; + +/// Selected models and seeds, and selected sources with loaders, whose relations this invocation produces. +#[pyfunction] +#[pyo3(signature = (*, scope, source_map))] +pub fn runtime_cursor_producer_names<'py>( + scope: &Bound<'py, PyAny>, + source_map: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = scope.py(); + let data = scope_data(scope)?; + let mut names: Vec<&str> = Vec::new(); + for (kind, name) in &data.selected { + match kind.as_str() { + "model" | "seed" => names.push(name), + "source" => { + let source: Bound<'py, PyAny> = mapping_get(source_map, name)?; + if !source.is_none() && !native_field(&source, "loader")?.is_none() { + names.push(name); + } + } + _ => {} + } + } + PyFrozenSet::new(py, names) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names_function.rs new file mode 100644 index 0000000000..03d89f796c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_inputs/runtime_cursor_producer_names_function.rs @@ -0,0 +1,10 @@ +//! `runtime_cursor_producer_names` as a function of `module`. + +use crate::planner_strategy::main::cursor_inputs::runtime_cursor_producer_names::runtime_cursor_producer_names; + +/// `runtime_cursor_producer_names` as a function of `module`. +pub fn runtime_cursor_producer_names_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(runtime_cursor_producer_names, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/classify_cursor_sql_type.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/classify_cursor_sql_type.rs new file mode 100644 index 0000000000..5c3153f9ce --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/classify_cursor_sql_type.rs @@ -0,0 +1,31 @@ +//! The `CursorType` a supported cross-adapter SQL type holds, or `None`. + +use crate::planner_changes::main::planner_enum::planner_enum; +use crate::planner_strategy::_helpers::cursor_types::log_parse_failure; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_analysis::planner_strategy::main::classify_cursor_sql_type::classify_cursor_sql_type as classify; +use sqlbuild_analysis::planner_strategy::models::CursorType; +use sqlbuild_python_base::records::types::RecordId; + +/// The `CursorType` a supported cross-adapter SQL type holds, or `None`. +pub fn classify_cursor_sql_type<'py>( + py: Python<'py>, + sql_type: &str, +) -> PyResult>> { + let detected: Option = match classify(sql_type) { + Ok(detected) => detected, + Err(failure) => { + log_parse_failure(py, &failure)?; + None + } + }; + detected + .map(|cursor_type| { + let value: &str = match cursor_type { + CursorType::Timestamp => "timestamp", + CursorType::Integer => "integer", + }; + planner_enum(py, RecordId::CursorType, value) + }) + .transpose() +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/mod.rs new file mode 100644 index 0000000000..67abd3e832 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/mod.rs @@ -0,0 +1,2 @@ +pub mod classify_cursor_sql_type; +pub mod register_cursor_types; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/register_cursor_types.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/register_cursor_types.rs new file mode 100644 index 0000000000..5f5507ec6b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/cursor_types/register_cursor_types.rs @@ -0,0 +1,10 @@ +//! Register the cursor types functions and classes. + +use crate::planner_strategy::_helpers::cursor_types::check_cursor_type_consistency; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the cursor types functions and classes. +pub fn register_cursor_types(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(check_cursor_type_consistency, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/mod.rs new file mode 100644 index 0000000000..b4626ea813 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/mod.rs @@ -0,0 +1,2 @@ +pub mod reject_capped_watermark_inputs; +pub mod resolve_lineage_reference; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/reject_capped_watermark_inputs.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/reject_capped_watermark_inputs.rs new file mode 100644 index 0000000000..01372664aa --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/reject_capped_watermark_inputs.rs @@ -0,0 +1,46 @@ +//! Reject capped producers feeding a microbatch model's watermark inputs. + +use crate::planner_strategy::_helpers::lineage::ProjectLineage; +use crate::planner_strategy::_helpers::lineage::lineage_error; +use crate::planner_strategy::_helpers::lineage::microbatch_flag; +use crate::planner_strategy::_helpers::lineage::references; +use crate::planner_strategy::_helpers::lineage::watermark_names; +use crate::planner_strategy::main::values::input_error::input_error; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_analysis::planner_strategy::main::capped_watermark_input::capped_watermark_input; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::native_host::models::ModelLookup; + +/// Reject capped producers feeding a microbatch model's watermark inputs. +pub fn reject_capped_watermark_inputs<'py>( + model: &Bound<'py, PyAny>, + models_by_name: ModelLookup, + functions_by_name: Option>, +) -> PyResult<()> { + let py: Python<'py> = model.py(); + if !microbatch_flag(model)? { + return Ok(()); + } + let functions_by_name: Bound<'py, PyAny> = match functions_by_name { + Some(functions) if functions.is_truthy()? => functions, + _ => PyDict::new(py).into_any(), + }; + let graph = ProjectLineage { + models_by_name, + functions_by_name, + }; + let name: String = model_name(model)?; + let inputs: Vec = watermark_names(model)?; + match capped_watermark_input(&graph, (&name, references(model)?), inputs) { + Ok(None) => Ok(()), + Ok(Some(capped)) => Err(input_error( + py, + ErrorId::CompileInput, + &capped.message(), + None, + )), + Err(failure) => Err(lineage_error(py, failure)), + } +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/resolve_lineage_reference.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/resolve_lineage_reference.rs new file mode 100644 index 0000000000..82d22edbd7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/lineage/resolve_lineage_reference.rs @@ -0,0 +1,26 @@ +//! The reference one named input resolves to through model and SQL function references. + +use crate::planner_strategy::_helpers::lineage::ProjectLineage; +use crate::planner_strategy::_helpers::lineage::lineage_error; +use crate::planner_strategy::_helpers::lineage::references; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_analysis::planner_strategy::main::lineage_reference::lineage_reference; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::native_host::models::ModelLookup; + +/// The reference one named input resolves to through model and SQL function references. +pub fn resolve_lineage_reference<'py>( + model: &Bound<'py, PyAny>, + input_name: &str, + models_by_name: ModelLookup, + functions_by_name: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = model.py(); + let graph = ProjectLineage { + models_by_name, + functions_by_name, + }; + let name: String = model_name(model)?; + lineage_reference(&graph, (&name, references(model)?), input_name) + .map_err(|failure| lineage_error(py, failure)) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs new file mode 100644 index 0000000000..8a05cc4946 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs @@ -0,0 +1,48 @@ +//! Apply CLI, project, and model batch limits in their documented precedence order. + +use crate::planner_strategy::_helpers::microbatch_limits::default_limit; +use crate::planner_strategy::_helpers::microbatch_limits::limited; +use crate::planner_strategy::_helpers::microbatch_limits::with_warnings; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::project::replace::replace; + +/// Apply CLI, project, and model batch limits in their documented precedence order. +#[pyfunction] +#[pyo3(signature = (*, entry, inputs))] +pub fn apply_configured_microbatch_limits<'py>( + entry: &Bound<'py, PyAny>, + inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = entry.py(); + let configured: Bound<'py, PyAny> = native_field(inputs, "max_microbatches")?; + let configured_action: Bound<'py, PyAny> = native_field(inputs, "microbatch_limit_action")?; + if native_field(inputs, "max_microbatches_is_override")?.is_truthy()? { + let error: Bound<'py, PyAny> = enum_member(py, RecordId::MicrobatchLimitAction, "ERROR")?; + return with_warnings(limited(entry, configured, error)?, Vec::new()); + } + let declared: Bound<'py, PyAny> = native_field(entry, "declared_microbatch_limit_action")?; + if declared.is_none() { + let maximum: Bound<'py, PyAny> = default_limit(entry, configured)?; + return with_warnings(limited(entry, maximum, configured_action)?, Vec::new()); + } + let mut warnings: Vec> = Vec::new(); + let mut entry: Bound<'py, PyAny> = entry.clone(); + if !configured.is_none() { + let (_, safety_warning) = limited(&entry, configured.clone(), configured_action.clone())?; + entry = replace( + &entry, + &[ + ("microbatch_safety_limit", configured), + ("microbatch_safety_limit_action", configured_action), + ], + )?; + warnings.extend(safety_warning); + } + let model_limit: Bound<'py, PyAny> = native_field(&entry, "microbatch_limit")?; + with_warnings(limited(&entry, model_limit, declared)?, warnings) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits_function.rs new file mode 100644 index 0000000000..10e8cd5641 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits_function.rs @@ -0,0 +1,10 @@ +//! `apply_configured_microbatch_limits` as a function of `module`. + +use crate::planner_strategy::main::microbatch_limits::apply_configured_microbatch_limits::apply_configured_microbatch_limits; + +/// `apply_configured_microbatch_limits` as a function of `module`. +pub fn apply_configured_microbatch_limits_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(apply_configured_microbatch_limits, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_microbatch_limit_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_microbatch_limit_function.rs new file mode 100644 index 0000000000..997c5de3cf --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_microbatch_limit_function.rs @@ -0,0 +1,10 @@ +//! `apply_microbatch_limit` as a function of `module`. + +use crate::planner_strategy::_helpers::microbatch_limits::apply_microbatch_limit; + +/// `apply_microbatch_limit` as a function of `module`. +pub fn apply_microbatch_limit_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(apply_microbatch_limit, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/mod.rs new file mode 100644 index 0000000000..102c9624e7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/mod.rs @@ -0,0 +1,3 @@ +pub mod apply_configured_microbatch_limits; +pub mod apply_configured_microbatch_limits_function; +pub mod apply_microbatch_limit_function; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/mod.rs new file mode 100644 index 0000000000..de713ce3e5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/mod.rs @@ -0,0 +1 @@ +pub mod register_microbatches; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/register_microbatches.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/register_microbatches.rs new file mode 100644 index 0000000000..c202e89d21 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/microbatches/register_microbatches.rs @@ -0,0 +1,11 @@ +//! Register the microbatches functions and classes. + +use crate::planner_strategy::_helpers::microbatches::microbatch_limit_warning; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the microbatches functions and classes. +pub fn register_microbatches(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(microbatch_limit_warning, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/mod.rs new file mode 100644 index 0000000000..66ad816139 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/mod.rs @@ -0,0 +1,10 @@ +pub mod audit_entries; +pub mod audits; +pub mod cursor_inputs; +pub mod cursor_types; +pub mod lineage; +pub mod microbatch_limits; +pub mod microbatches; +pub mod model_entries; +pub mod strategy; +pub mod values; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/assemble_model_plan_entry.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/assemble_model_plan_entry.rs new file mode 100644 index 0000000000..4a49ef367c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/assemble_model_plan_entry.rs @@ -0,0 +1,14 @@ +//! Build a model plan entry and its warnings from resolved plan decisions. + +use crate::planner_strategy::_helpers::model_entries; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyTuple; +use sqlbuild_python_project::planner_strategy::models::ModelEntryDecisions; + +/// Build a model plan entry and its warnings from resolved plan decisions. +#[inline] +pub fn assemble_model_plan_entry<'py>( + request: &ModelEntryDecisions<'py>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyTuple>)> { + model_entries::assemble_model_plan_entry(request) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/extract_seed_columns.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/extract_seed_columns.rs new file mode 100644 index 0000000000..9c765aaa4b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/extract_seed_columns.rs @@ -0,0 +1,10 @@ +//! Declared seed columns with types, as `ColumnInfo`. + +use crate::planner_strategy::_helpers::model_entries::schema_columns; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyTuple; + +/// Declared seed columns with types, as `ColumnInfo`. +pub fn extract_seed_columns<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + schema_columns(seed, true) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/mod.rs new file mode 100644 index 0000000000..4725a86594 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/mod.rs @@ -0,0 +1,4 @@ +pub mod assemble_model_plan_entry; +pub mod extract_seed_columns; +pub mod resolve_microbatch_lookback; +pub mod resolve_microbatch_lookback_function; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback.rs new file mode 100644 index 0000000000..2992ac7c77 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback.rs @@ -0,0 +1,15 @@ +//! Resolve explicit or default lookback for one microbatch model. + +use crate::planner_strategy::_helpers::model_entries::lookback_of; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; + +/// Resolve explicit or default lookback for one microbatch model. +#[pyfunction] +#[pyo3(signature = (*, model, effective_grain = None))] +pub fn resolve_microbatch_lookback( + model: &Bound<'_, PyAny>, + effective_grain: Option, +) -> PyResult> { + lookback_of(model, effective_grain) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback_function.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback_function.rs new file mode 100644 index 0000000000..a55342d16d --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/model_entries/resolve_microbatch_lookback_function.rs @@ -0,0 +1,10 @@ +//! `resolve_microbatch_lookback` as a function of `module`. + +use crate::planner_strategy::main::model_entries::resolve_microbatch_lookback::resolve_microbatch_lookback; + +/// `resolve_microbatch_lookback` as a function of `module`. +pub fn resolve_microbatch_lookback_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(resolve_microbatch_lookback, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/materialization_of.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/materialization_of.rs new file mode 100644 index 0000000000..45a13d553e --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/materialization_of.rs @@ -0,0 +1,18 @@ +//! The `MaterializationType` of a model's `materialized` config: table when unset, custom when unknown. + +use crate::planner_changes::main::planner_enum::planner_enum; +use crate::planner_strategy::_helpers::values::{materialization_value, model_materialization}; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::RecordId; + +/// The `MaterializationType` of a model's `materialized` config: table when unset, custom when unknown. +#[pyfunction(name = "get_materialization_type")] +#[pyo3(signature = (model))] +pub fn materialization_of<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + planner_enum( + model.py(), + RecordId::MaterializationType, + materialization_value(model_materialization(model)?), + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/mod.rs new file mode 100644 index 0000000000..d6a3768b49 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/mod.rs @@ -0,0 +1,3 @@ +pub mod materialization_of; +pub mod register_strategy; +pub mod resolve_model_plan_action; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs new file mode 100644 index 0000000000..57596372a8 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs @@ -0,0 +1,16 @@ +//! Register the strategy functions and classes. + +use crate::planner_strategy::_helpers::strategy::build_model_warnings; +use crate::planner_strategy::_helpers::strategy::resolve_schema_actions; +use crate::planner_strategy::main::strategy::materialization_of::materialization_of; +use crate::planner_strategy::main::strategy::resolve_model_plan_action::resolve_model_plan_action; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the strategy functions and classes. +pub fn register_strategy(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(materialization_of, module)?)?; + module.add_function(wrap_pyfunction!(resolve_model_plan_action, module)?)?; + module.add_function(wrap_pyfunction!(resolve_schema_actions, module)?)?; + module.add_function(wrap_pyfunction!(build_model_warnings, module)?) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/resolve_model_plan_action.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/resolve_model_plan_action.rs new file mode 100644 index 0000000000..9d875a1b3b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/resolve_model_plan_action.rs @@ -0,0 +1,55 @@ +//! The plan action and reason for one model. + +use crate::planner_changes::main::planner_enum::planner_enum; +use crate::planner_strategy::_helpers::strategy::action_value; +use crate::planner_strategy::_helpers::strategy::reason_value; +use crate::planner_strategy::_helpers::values::{ + change_facts, config_value, model_materialization, optional_string, +}; +use crate::planner_strategy::main::values::input_error::input_error; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyBool, PyTuple}; +use sqlbuild_analysis::planner_strategy::main::model_plan_action::model_plan_action; +use sqlbuild_analysis::planner_strategy::models::ModelStrategy; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_base::records::types::RecordId; + +/// The plan action and reason for one model. +#[pyfunction] +#[pyo3(signature = (*, model, change_result, full_refresh))] +pub fn resolve_model_plan_action<'py>( + model: &Bound<'py, PyAny>, + change_result: &Bound<'py, PyAny>, + full_refresh: bool, +) -> PyResult> { + let py: Python<'py> = model.py(); + let enabled: Bound<'py, PyAny> = config_value(model, "enabled")?; + let strategy = ModelStrategy { + name: + sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name( + model, + )?, + disabled: enabled.is_instance_of::() && !enabled.is_truthy()?, + materialization: model_materialization(model)?, + incremental_strategy: optional_string(&config_value(model, "incremental_strategy")?)?, + }; + let (action, reason) = + model_plan_action(&strategy, &change_facts(change_result)?, full_refresh).map_err( + |error| { + input_error( + py, + ErrorId::PlannerInput, + &error.message(), + Some(error.code()), + ) + }, + )?; + PyTuple::new( + py, + [ + planner_enum(py, RecordId::PlanAction, action_value(action))?, + planner_enum(py, RecordId::PlanReason, reason_value(reason))?, + ], + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs new file mode 100644 index 0000000000..22fc32fc49 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs @@ -0,0 +1,21 @@ +//! The native error `class(message, code=code)`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; + +/// The native error `class(message, code=code)`. +pub fn input_error(py: Python<'_>, class: ErrorId, message: &str, code: Option<&str>) -> PyErr { + let raised: PyResult> = error_class(py, class).and_then(|class| { + let options: Bound<'_, PyDict> = PyDict::new(py); + if let Some(code) = code { + options.set_item("code", code)?; + } + class.call((message,), Some(&options)) + }); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/values/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/mod.rs new file mode 100644 index 0000000000..a67b53e73f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/mod.rs @@ -0,0 +1,2 @@ +pub mod input_error; +pub mod text; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/values/text.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/text.rs new file mode 100644 index 0000000000..6426d587e9 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/values/text.rs @@ -0,0 +1,8 @@ +//! `str(value)` of a Python string or string enum. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +/// `str(value)` of a Python string or string enum. +pub fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.extract() +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/mod.rs b/crates/sqlbuild-python-planner/src/planner_strategy/mod.rs new file mode 100644 index 0000000000..acc20e2651 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_strategy/mod.rs @@ -0,0 +1,4 @@ +//! Planner strategy for model entries: audits, cursor inputs and types, lineage and microbatches. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs index 877eb665ee..bf0e2ea08b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs @@ -1,18 +1,19 @@ //! Cursor MIN/MAX inspection of selected incremental models and their watermark inputs. -use crate::bindings::_helpers::planner_resolution::cursor_policies::resolve_start_cursor_config; -use crate::bindings::_helpers::planner_resolution::cursor_windows::normalize_cursor_snapshot_grain; -use crate::bindings::_helpers::planner_resolution::cursor_windows::resolve_effective_timestamp_grain; -use crate::bindings::_helpers::planner_resolution::maximum_safety::maximum_allowed_start; -use crate::bindings::_helpers::planner_strategy::lineage::resolve_lineage_reference; -use crate::bindings::_helpers::planner_warehouse::constants::{ +use crate::planner_resolution::main::cursor::normalize_cursor_snapshot_grain::normalize_cursor_snapshot_grain; +use crate::planner_resolution::main::cursor::resolve_effective_timestamp_grain::resolve_effective_timestamp_grain; +use crate::planner_resolution::main::cursor::resolve_start_cursor_config::resolve_start_cursor_config; +use crate::planner_resolution::main::safety::maximum_allowed_start::maximum_allowed_start; +use crate::planner_strategy::main::lineage::resolve_lineage_reference::resolve_lineage_reference; +use crate::planner_warehouse::_helpers::inspection::snapshot::inspection_concurrency; +use crate::planner_warehouse::_helpers::inspection::watermarks::validate_watermark_column; +use crate::planner_warehouse::constants::{ CURSOR_ALGEBRA_ERROR, CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, DATE_VALUE, LOG_DEBUG_EVENT, PLANNER_LOGGER, QUERY_RECORD, RECORD_QUERY, RUN_BOUNDED, RUN_RECORDED_QUERY, }; -use crate::bindings::_helpers::planner_warehouse::snapshot::{ - SnapshotRequest, inspection_concurrency, -}; -use crate::bindings::_helpers::planner_warehouse::watermarks::validate_watermark_column; +use crate::planner_warehouse::main::inspection::adapter_cursor_sql::adapter_cursor_sql; +use crate::planner_warehouse::main::inspection::bound_task::bound_task; +use crate::planner_warehouse::models::SnapshotRequest; use pyo3::IntoPyObject; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; @@ -21,7 +22,6 @@ use sqlbuild_analysis::planner_changes::constants::FORWARD_ONLY_POLICY; use sqlbuild_analysis::planner_warehouse::main::cursor_bounds::cursor_bounds; use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_query_groups; use sqlbuild_analysis::planner_warehouse::main::cursor_query_identity::cursor_query_identity; -use sqlbuild_analysis::planner_warehouse::main::cursor_query_sql::cursor_query_sql; use sqlbuild_analysis::planner_warehouse::models::{CursorTagRequest, PhysicalCursorQuery}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -55,14 +55,13 @@ use sqlbuild_python_project::planner_phases::main::text::text; use sqlbuild_python_project::planner_phases::main::tuple::tuple; use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; -use std::cell::RefCell; use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; /// The adapter method that renders one cursor bound expression as text. -const CURSOR_TEXT_METHOD: &str = "render_cursor_text"; +pub(crate) const CURSOR_TEXT_METHOD: &str = "render_cursor_text"; /// Selection and policy inputs for cursor warehouse inspection. pub(crate) struct CursorInputs<'py> { @@ -135,7 +134,7 @@ fn parse<'py>( sqlbuild_python_front::cursor::main::scalars::parse::parse(raw, &text(py, cursor_type)) } -fn render<'py>(value: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn render<'py>(value: &Bound<'py, PyAny>) -> PyResult> { object(value.py(), render_value(value)?) } @@ -579,7 +578,7 @@ fn physical_queries(models: &[CursorModel<'_>]) -> Vec { } /// Run one physical cursor statement; a failure leaves only that relation unavailable. -fn read_cursor_query<'py>( +pub(crate) fn read_cursor_query<'py>( py: Python<'py>, sql: &str, (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -667,27 +666,6 @@ fn report_cursor_query( ) } -pub(crate) fn bound_task<'py>( - py: Python<'py>, - sql: String, - (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - let connection: Py = connection.clone().unbind(); - let execute: Py = execute.clone().unbind(); - Ok(PyCFunction::new_closure( - py, - None, - None, - move |args: &Bound<'_, PyTuple>, - _kwargs: Option<&Bound<'_, PyDict>>| - -> PyResult> { - let py = args.py(); - Ok(read_cursor_query(py, &sql, (connection.bind(py), execute.bind(py)))?.unbind()) - }, - )? - .into_any()) -} - fn completion_reporter<'py>( py: Python<'py>, queries: Vec, @@ -743,34 +721,6 @@ fn fan_out<'py>(query: &PhysicalCursorQuery, row: &Bound<'py, PyAny>) -> PyResul Ok(values) } -/// The query's MIN/MAX statement, each bound read as text the way the adapter renders it. -pub(crate) fn adapter_cursor_sql( - adapter: &Bound<'_, PyAny>, - query: &PhysicalCursorQuery, -) -> PyResult { - let py = adapter.py(); - let render = native_field(adapter, CURSOR_TEXT_METHOD)?; - let failure: RefCell> = RefCell::new(None); - let sql: String = cursor_query_sql(query, &|expression: &str| { - call_with( - &render, - &[ - ("expression", text(py, expression)), - ("cursor_type", text(py, &query.cursor_type)), - ], - ) - .and_then(|rendered| rendered.extract::()) - .unwrap_or_else(|error| { - failure.replace(Some(error)); - String::new() - }) - }); - match failure.into_inner() { - Some(error) => Err(error), - None => Ok(sql), - } -} - fn execute_queries<'py>( request: &SnapshotRequest<'py>, queries: &[PhysicalCursorQuery], diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs new file mode 100644 index 0000000000..58eb73eae7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs @@ -0,0 +1,105 @@ +//! Migration fingerprints computed at most once per model definition and reused across runs. + +use crate::planner_warehouse::models::FingerprintWork; +use pyo3::PyRefMut; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyFloat; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::imported::imported; +use std::hash::{DefaultHasher, Hash, Hasher}; + +pub(crate) const CACHE_DIRECTORY_NAME: &str = "migration-fingerprints-v1"; +pub(crate) const CACHE_DATABASE_NAME: &str = "migration-fingerprints.sqlite3"; +const SQLITE_TIMEOUT_SECONDS: f64 = 0.1; +pub(crate) const CREATE_CACHE_TABLE_SQL: &str = "\nCREATE TABLE IF NOT EXISTS migration_fingerprint (\n cache_key TEXT PRIMARY KEY,\n payload TEXT NOT NULL\n)\n"; +pub(crate) const INSERT_SQL: &str = + "INSERT OR REPLACE INTO migration_fingerprint (cache_key, payload) VALUES (?, ?)"; +pub(crate) const SELECT_SQL: &str = "SELECT payload FROM migration_fingerprint WHERE cache_key = ?"; + +/// A fingerprint request: query SQL, metadata JSON, dialect and visible renames. +#[derive(Debug, Hash, PartialEq, Eq)] +pub(crate) struct MemoKey { + pub(crate) query_sql: String, + pub(crate) metadata_json: String, + pub(crate) dialect: Option, + pub(crate) visible_renames: Vec<(String, String)>, +} + +static FINGERPRINT_WORK: PyOnceLock> = PyOnceLock::new(); + +fn fingerprint_work(py: Python<'_>) -> PyResult> { + Ok(FINGERPRINT_WORK + .get_or_try_init(py, || Py::new(py, FingerprintWork::default()))? + .bind(py) + .borrow_mut()) +} + +pub(crate) fn count_computation(py: Python<'_>, key: &MemoKey) -> PyResult<()> { + let mut hasher = DefaultHasher::new(); + key.hash(&mut hasher); + let mut work = fingerprint_work(py)?; + work.computations += 1; + work.distinct.insert(hasher.finish()); + Ok(()) +} + +/// Report migration fingerprints computed and distinct requests among them, optionally resetting. +#[pyfunction] +#[pyo3(signature = (*, reset = false))] +pub(crate) fn migration_fingerprint_work_info( + py: Python<'_>, + reset: bool, +) -> PyResult<(u64, usize)> { + let mut work = fingerprint_work(py)?; + let info = (work.computations, work.distinct.len()); + if reset { + *work = FingerprintWork::default(); + } + Ok(info) +} + +pub(crate) fn sqlite_error(py: Python<'_>, error: &PyErr) -> PyResult { + Ok(error.is_instance(py, &imported(py, "sqlite3", "DatabaseError")?)) +} + +pub(crate) fn close(py: Python<'_>, connection: Option>) -> PyResult<()> { + let Some(connection) = connection else { + return Ok(()); + }; + match connection.bind(py).call_method0("close") { + Ok(_) => Ok(()), + Err(error) if sqlite_error(py, &error)? => Ok(()), + Err(error) => Err(error), + } +} + +pub(crate) fn connect<'py>(path: &Bound<'py, PyAny>) -> PyResult> { + let py = path.py(); + call_with( + &imported(py, "sqlite3", "connect")?, + &[ + ("database", path.clone()), + ( + "timeout", + PyFloat::new(py, SQLITE_TIMEOUT_SECONDS).into_any(), + ), + ], + ) +} + +pub(crate) fn package_version(py: Python<'_>, package: &str) -> PyResult { + match imported(py, "importlib.metadata", "version")?.call1((package,)) { + Ok(version) => version.extract(), + Err(error) + if error.is_instance( + py, + &imported(py, "importlib.metadata", "PackageNotFoundError")?, + ) => + { + Ok("unknown".to_owned()) + } + Err(error) => Err(error), + } +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/mod.rs new file mode 100644 index 0000000000..1c57d002cb --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod cursors; +pub(crate) mod fingerprint_cache; +pub(crate) mod snapshot; +pub(crate) mod watermarks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs index a16d161f3a..5919658d1a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs @@ -1,11 +1,12 @@ //! Gather a frozen point-in-time warehouse snapshot for planning. -use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; -use crate::bindings::_helpers::planner_warehouse::constants::{ +use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; +use crate::planner_warehouse::_helpers::inspection::cursors::{CursorInputs, cursor_snapshots}; +use crate::planner_warehouse::constants::{ ACTIVE_CATALOG, FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, MIGRATE_FROM_CONFIG_KEY, RELATIONS_MAIN_COLUMNS, RELATIONS_MAIN_LIST, }; -use crate::bindings::_helpers::planner_warehouse::cursors::{CursorInputs, cursor_snapshots}; +use crate::planner_warehouse::models::SnapshotRequest; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -48,20 +49,6 @@ use sqlbuild_python_project::planner_scope::main::build_upstream_intermediate_so use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; use std::collections::{BTreeSet, HashSet}; -/// Everything one snapshot gather needs. -pub(crate) struct SnapshotRequest<'py> { - pub(crate) project: Bound<'py, PyAny>, - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) connection: Bound<'py, PyAny>, - pub(crate) execute: Bound<'py, PyAny>, - pub(crate) selected_keys: Bound<'py, PyAny>, - pub(crate) full_refresh: bool, - pub(crate) full_refresh_model_names: Bound<'py, PyAny>, - pub(crate) on_progress: Bound<'py, PyAny>, - pub(crate) deferred_locations: Bound<'py, PyAny>, - pub(crate) cursor_scope: Bound<'py, PyAny>, -} - #[derive(Default)] struct StateSchemas { fingerprints: BTreeSet, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs index ea9d541ea8..c22cfb4a51 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs @@ -1,7 +1,7 @@ //! Validate reliable watermark column contracts before warehouse inspection. -use crate::bindings::_helpers::planner_strategy::cursor_types::classify_cursor_sql_type; -use crate::bindings::_helpers::planner_warehouse::constants::{ +use crate::planner_strategy::main::cursor_types::classify_cursor_sql_type::classify_cursor_sql_type; +use crate::planner_warehouse::constants::{ CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, WATERMARK_CONTRACT_CODE, }; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/mod.rs new file mode 100644 index 0000000000..349ca8f38c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod inspection; +pub(crate) mod sources; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/mod.rs new file mode 100644 index 0000000000..9ec65b96fd --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod semantic_sources; +pub(crate) mod source_columns; +pub(crate) mod source_deferral; +pub(crate) mod source_freshness; +pub(crate) mod source_tables; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs index d0a0ae05f0..eabd9cf952 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs @@ -1,7 +1,7 @@ //! Complete missing source metadata in bounded, database-grouped batches. -use crate::bindings::_helpers::planner_warehouse::constants::ACTIVE_CATALOG; -use crate::bindings::_helpers::planner_warehouse::snapshot::{columns_for, list_relations}; +use crate::planner_warehouse::_helpers::inspection::snapshot::{columns_for, list_relations}; +use crate::planner_warehouse::constants::ACTIVE_CATALOG; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs index 1047e704e8..accc47be4b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs @@ -1,13 +1,12 @@ //! Warehouse source relation listing and column gathering for planning. -use crate::bindings::_helpers::observations::source_capture::capture_source_columns; -use crate::bindings::_helpers::planner_warehouse::constants::{ +use crate::observations::main::capture_source_columns::capture_source_columns; +use crate::planner_warehouse::_helpers::inspection::snapshot::{columns_for, list_relations}; +use crate::planner_warehouse::constants::{ COLUMN_INFO, METADATA_NAME_FILTER_LIMIT, RELATION_LOOKUP, }; -use crate::bindings::_helpers::planner_warehouse::snapshot::{columns_for, list_relations}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::pyfunction; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -227,24 +226,3 @@ pub(crate) fn source_inspection<'py>( } Ok((result, frozenset(py, &listed)?)) } - -pub(crate) fn gather_source_inspection<'py>( - project: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, - source_entries: Option>, -) -> PyResult> { - let (columns, listed) = source_inspection(project, (adapter, connection), source_entries)?; - Ok(PyTuple::new(project.py(), [columns.into_any(), listed])?.into_any()) -} - -#[pyfunction] -#[pyo3(signature = (*, project, adapter, connection, source_entries=None))] -pub(crate) fn gather_source_columns<'py>( - project: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, - source_entries: Option>, -) -> PyResult> { - Ok(source_inspection(project, (adapter, connection), source_entries)?.0) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs index 5b0f7ff821..30f0337351 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs @@ -2,9 +2,7 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::planner_warehouse::constants::{ - MODEL_HOOK_CONFIG_KEYS, SQL_HOOK_ENTRY, -}; +use crate::planner_warehouse::constants::{MODEL_HOOK_CONFIG_KEYS, SQL_HOOK_ENTRY}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; @@ -326,30 +324,3 @@ pub(crate) fn build_source_read_map<'py>( } Ok(result) } - -/// Add declared project sources that no selected SQL resource reads to a source map. -pub(crate) fn with_declared_source_reads<'py>( - project: &Bound<'py, PyAny>, - source_map: &Bound<'py, PyDict>, - source_names: &Bound<'py, PyAny>, -) -> PyResult> { - let mut missing: BTreeSet<_> = BTreeSet::new(); - for name in source_names.try_iter()? { - let name: String = name?.extract()?; - if !source_map.contains(&name)? { - missing.insert(name); - } - } - if missing.is_empty() { - return Ok(source_map.clone()); - } - let result = source_map.copy()?; - for source in sources(project)?.try_iter()? { - let entry = source_entry(&source?)?; - let name: String = native_field(&entry, "name")?.extract()?; - if missing.contains(&name) { - result.set_item(name, entry)?; - } - } - Ok(result) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs similarity index 70% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs index 5c8143ab3b..5a3b9c02d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs @@ -1,76 +1,27 @@ //! Direct planner source freshness observation and the warnings for unknown freshness. -use crate::bindings::_helpers::freshness::planning::PREVIEW_MESSAGE; -use crate::bindings::_helpers::freshness::planning::{ - build_direct_source_freshness_planning_result, build_direct_source_freshness_propagation_result, -}; -use crate::bindings::_helpers::planner_warehouse::constants::RUN_SCOPE; -use crate::bindings::_helpers::planner_warehouse::snapshot::{destination_nodes, lower}; +use crate::freshness::main::build_direct_source_freshness_planning_result_function::build_direct_source_freshness_planning_result_function; +use crate::freshness::main::build_direct_source_freshness_propagation_result_function::build_direct_source_freshness_propagation_result_function; +use crate::planner_warehouse::_helpers::inspection::snapshot::{destination_nodes, lower}; +use crate::planner_warehouse::constants::RUN_SCOPE; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyBool, PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_scope::constants::MODEL; -use sqlbuild_analysis::planner_warehouse::main::freshness_unknown_message::freshness_unknown_message; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; -use sqlbuild_python_project::planner_phases::main::native::native; use sqlbuild_python_project::planner_phases::main::replace::replace; use sqlbuild_python_project::planner_phases::main::text::text; use sqlbuild_python_project::planner_phases::main::tuple::tuple; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::BTreeSet; -/// One warning per reason that left configured source freshness unknown. -pub(crate) fn source_freshness_unknown_warnings<'py>( - source_freshness: &Bound<'py, PyAny>, -) -> PyResult> { - let py = source_freshness.py(); - let mut reasons: Vec<(String, Vec)> = Vec::new(); - for unknown in native_field(source_freshness, "unknown_sources")? - .call_method0("values")? - .try_iter()? - { - let unknown = unknown?; - let reason: String = native_field(&unknown, "reason")?.str()?.extract()?; - if freshness_unknown_message(&reason, &[]).is_none() - || native_field(&unknown, "message")?.eq(PREVIEW_MESSAGE)? - { - continue; - } - let name: String = native_field(&unknown, "source_name")?.extract()?; - match reasons.iter_mut().find(|(existing, _)| *existing == reason) { - Some((_, names)) => names.push(name), - None => reasons.push((reason, vec![name])), - } - } - let mut warnings: Vec<_> = Vec::new(); - for (reason, names) in reasons { - if let Some(message) = freshness_unknown_message(&reason, &names) { - warnings.push(native( - py, - RecordId::PlanWarning, - &[ - ("model_name", py.None().into_bound(py)), - ( - "severity", - enum_member(py, RecordId::WarningSeverity, "WARNING")?, - ), - ("message", text(py, &message)), - ], - )?); - } - } - tuple(py, warnings) -} - fn state_database<'py>(project: &Bound<'py, PyAny>) -> PyResult> { for node in destination_nodes(project)? { let database = native_field(&sqlbuild_python_project::resource_views::main::resource_reads::destination::destination(&node)?, "database")?; @@ -173,7 +124,7 @@ pub(crate) fn build_planner_source_freshness_result<'py>( &host_entry(py, RUN_SCOPE)?.call1((native_field(project, "run_id")?,))?, |_| { call_with( - wrap_pyfunction!(build_direct_source_freshness_planning_result, py)?.as_any(), + build_direct_source_freshness_planning_result_function(py)?.as_any(), &[ ("adapter", adapter.clone()), ("connection", connection.clone()), @@ -199,7 +150,7 @@ pub(crate) fn build_planner_source_freshness_result<'py>( }, )?; let propagation = call_with( - wrap_pyfunction!(build_direct_source_freshness_propagation_result, py)?.as_any(), + build_direct_source_freshness_propagation_result_function(py)?.as_any(), &[ ("source_freshness", result.clone()), ("scope", scope.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs index ed0277c229..c808eadbc4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs @@ -1,6 +1,6 @@ //! Selected-scope validation that external source tables exist in the warehouse. -use crate::bindings::_helpers::planner_warehouse::constants::{ +use crate::planner_warehouse::constants::{ MISSING_SOURCE_TABLE_CODE, RUN_BOUNDED, WILDCARD_TABLE_CHARACTER, }; use pyo3::IntoPyObject; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs index e684661f10..e60f0d0305 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs @@ -1,31 +1,53 @@ -//! Python modules and fixed names of warehouse inspection for planning. +//! Public constants of planner warehouse. use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; - use sqlbuild_python_base::records::types::RecordId; pub(crate) const RELATIONS_MAIN_LIST: AdapterHost = AdapterHost::ListRelationsForInspection; + pub(crate) const RELATIONS_MAIN_COLUMNS: AdapterHost = AdapterHost::GetColumnsForInspection; + pub(crate) const ACTIVE_CATALOG: AdapterHost = AdapterHost::ActiveInspectionCatalog; -pub(crate) const RUN_BOUNDED: AdapterHost = AdapterHost::RunBoundedInspections; + +pub const RUN_BOUNDED: AdapterHost = AdapterHost::RunBoundedInspections; + pub(crate) const RECORD_QUERY: AdapterHost = AdapterHost::RecordInspectionQuery; + pub(crate) const RUN_RECORDED_QUERY: AdapterHost = AdapterHost::RunRecordedInspectionQuery; + pub(crate) const QUERY_RECORD: AdapterHost = AdapterHost::InspectionQueryRecord; + pub(crate) const COLUMN_INFO: AdapterHost = AdapterHost::ColumnInfo; + pub(crate) const RELATION_LOOKUP: AdapterHost = AdapterHost::RelationLookup; + pub(crate) const LOG_DEBUG_EVENT: ExecutorHost = ExecutorHost::LogDebugEvent; + pub(crate) const RUN_SCOPE: ExecutorHost = ExecutorHost::RunScope; + pub(crate) const SQL_HOOK_ENTRY: RecordId = RecordId::SqlHookEntry; + pub(crate) const CURSOR_ALGEBRA_ERROR: (&str, &str) = ("sqlbuild.cursor_algebra.exceptions", "CursorAlgebraError"); + pub(crate) const DATE_VALUE: RecordId = RecordId::DateValue; + pub(crate) const PLANNER_LOGGER: &str = "sqlbuild.planner"; + pub(crate) const METADATA_NAME_FILTER_LIMIT: usize = 250; + pub(crate) const FUNCTION_NODE_TYPES: [&str; 2] = ["udf", "table_fn"]; + pub(crate) const WILDCARD_TABLE_CHARACTER: &str = "*"; + pub(crate) const MIGRATE_FROM_CONFIG_KEY: &str = "migrate_from"; + pub(crate) const MODEL_HOOK_CONFIG_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; + pub(crate) const CURSOR_TYPE_TIMESTAMP: &str = "timestamp"; + pub(crate) const CURSOR_TYPE_INTEGER: &str = "integer"; + pub(crate) const MISSING_SOURCE_TABLE_CODE: &str = "S405"; + pub(crate) const WATERMARK_CONTRACT_CODE: &str = "S302"; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/adapter_cursor_sql.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/adapter_cursor_sql.rs new file mode 100644 index 0000000000..e079335ca7 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/adapter_cursor_sql.rs @@ -0,0 +1,38 @@ +//! The query's MIN/MAX statement, each bound read as text the way the adapter renders it. + +use crate::planner_warehouse::_helpers::inspection::cursors::CURSOR_TEXT_METHOD; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; +use sqlbuild_analysis::planner_warehouse::main::cursor_query_sql::cursor_query_sql; +use sqlbuild_analysis::planner_warehouse::models::PhysicalCursorQuery; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_phases::main::text::text; +use std::cell::RefCell; + +/// The query's MIN/MAX statement, each bound read as text the way the adapter renders it. +pub fn adapter_cursor_sql( + adapter: &Bound<'_, PyAny>, + query: &PhysicalCursorQuery, +) -> PyResult { + let py = adapter.py(); + let render = native_field(adapter, CURSOR_TEXT_METHOD)?; + let failure: RefCell> = RefCell::new(None); + let sql: String = cursor_query_sql(query, &|expression: &str| { + call_with( + &render, + &[ + ("expression", text(py, expression)), + ("cursor_type", text(py, &query.cursor_type)), + ], + ) + .and_then(|rendered| rendered.extract::()) + .unwrap_or_else(|error| { + failure.replace(Some(error)); + String::new() + }) + }); + match failure.into_inner() { + Some(error) => Err(error), + None => Ok(sql), + } +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/bound_task.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/bound_task.rs new file mode 100644 index 0000000000..30c54090f5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/bound_task.rs @@ -0,0 +1,27 @@ +//! Bound task. + +use crate::planner_warehouse::_helpers::inspection::cursors::read_cursor_query; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::types::{PyCFunction, PyDict, PyTuple}; + +/// Bound task. +pub fn bound_task<'py>( + py: Python<'py>, + sql: String, + (connection, execute): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let connection: Py = connection.clone().unbind(); + let execute: Py = execute.clone().unbind(); + Ok(PyCFunction::new_closure( + py, + None, + None, + move |args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>| + -> PyResult> { + let py = args.py(); + Ok(read_cursor_query(py, &sql, (connection.bind(py), execute.bind(py)))?.unbind()) + }, + )? + .into_any()) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_redirected_cursor_snapshots.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_redirected_cursor_snapshots.rs new file mode 100644 index 0000000000..62154cdfdf --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_redirected_cursor_snapshots.rs @@ -0,0 +1,14 @@ +//! Cursor snapshots of models whose history lives in another relation or column. + +use crate::planner_warehouse::_helpers::inspection::snapshot; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyDict as KeywordDict; + +/// Cursor snapshots of models whose history lives in another relation or column. +#[inline] +pub fn gather_redirected_cursor_snapshots<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + snapshot::gather_redirected_cursor_snapshots(py, options) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_warehouse_snapshot_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_warehouse_snapshot_function.rs new file mode 100644 index 0000000000..7d960618c4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/gather_warehouse_snapshot_function.rs @@ -0,0 +1,10 @@ +//! `gather_warehouse_snapshot` as a function of `module`. + +use crate::planner_warehouse::_helpers::inspection::snapshot::gather_warehouse_snapshot; + +/// `gather_warehouse_snapshot` as a function of `module`. +pub fn gather_warehouse_snapshot_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(gather_warehouse_snapshot, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/migration_fingerprint_work_info_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/migration_fingerprint_work_info_function.rs new file mode 100644 index 0000000000..7fd8b07545 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/migration_fingerprint_work_info_function.rs @@ -0,0 +1,10 @@ +//! `migration_fingerprint_work_info` as a function of `module`. + +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::migration_fingerprint_work_info; + +/// `migration_fingerprint_work_info` as a function of `module`. +pub fn migration_fingerprint_work_info_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(migration_fingerprint_work_info, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/mod.rs new file mode 100644 index 0000000000..b6ff3dc93c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/mod.rs @@ -0,0 +1,8 @@ +pub mod adapter_cursor_sql; +pub mod bound_task; +pub mod gather_redirected_cursor_snapshots; +pub mod gather_warehouse_snapshot_function; +pub mod migration_fingerprint_work_info_function; +pub mod planner_metadata_name_filter_function; +pub mod warehouse_snapshot; +pub mod watermark_type_is_compatible_function; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/planner_metadata_name_filter_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/planner_metadata_name_filter_function.rs new file mode 100644 index 0000000000..193c3dd1b4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/planner_metadata_name_filter_function.rs @@ -0,0 +1,10 @@ +//! `planner_metadata_name_filter` as a function of `module`. + +use crate::planner_warehouse::_helpers::inspection::snapshot::planner_metadata_name_filter; + +/// `planner_metadata_name_filter` as a function of `module`. +pub fn planner_metadata_name_filter_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(planner_metadata_name_filter, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/warehouse_snapshot.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/warehouse_snapshot.rs new file mode 100644 index 0000000000..dbd37a07b1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/warehouse_snapshot.rs @@ -0,0 +1,11 @@ +//! Gather relations, columns, fingerprints and cursor bounds of the planned target schemas. + +use crate::planner_warehouse::_helpers::inspection::snapshot; +use crate::planner_warehouse::models::SnapshotRequest; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Gather relations, columns, fingerprints and cursor bounds of the planned target schemas. +#[inline] +pub fn warehouse_snapshot<'py>(request: &SnapshotRequest<'py>) -> PyResult> { + snapshot::warehouse_snapshot(request) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/watermark_type_is_compatible_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/watermark_type_is_compatible_function.rs new file mode 100644 index 0000000000..bf5fecd895 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/inspection/watermark_type_is_compatible_function.rs @@ -0,0 +1,10 @@ +//! `watermark_type_is_compatible` as a function of `module`. + +use crate::planner_warehouse::_helpers::inspection::watermarks::watermark_type_is_compatible; + +/// `watermark_type_is_compatible` as a function of `module`. +pub fn watermark_type_is_compatible_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(watermark_type_is_compatible, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/mod.rs new file mode 100644 index 0000000000..87c68376c1 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/mod.rs @@ -0,0 +1,2 @@ +pub mod inspection; +pub mod sources; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result.rs new file mode 100644 index 0000000000..a5f1839f35 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result.rs @@ -0,0 +1,20 @@ +//! The source freshness evaluation of one inspection scope against the warehouse. + +use crate::planner_warehouse::_helpers::sources::source_freshness; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// The source freshness evaluation of one inspection scope against the warehouse. +#[inline] +pub fn build_planner_source_freshness_result<'py>( + (project, scope): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + relations: &Bound<'py, PyAny>, + freshness_state_schemas: &Bound<'py, PyAny>, +) -> PyResult> { + source_freshness::build_planner_source_freshness_result( + (project, scope), + (adapter, connection), + relations, + freshness_state_schemas, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result_of_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result_of_function.rs new file mode 100644 index 0000000000..ef83913e24 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_planner_source_freshness_result_of_function.rs @@ -0,0 +1,10 @@ +//! `build_planner_source_freshness_result_of` as a function of `module`. + +use crate::planner_warehouse::_helpers::sources::source_freshness::build_planner_source_freshness_result_of; + +/// `build_planner_source_freshness_result_of` as a function of `module`. +pub fn build_planner_source_freshness_result_of_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_planner_source_freshness_result_of, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_source_read_map.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_source_read_map.rs new file mode 100644 index 0000000000..2ea35fdb8f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/build_source_read_map.rs @@ -0,0 +1,14 @@ +//! Resolve source reads for selected SQL, or for `read_source_names` when given. + +use crate::planner_warehouse::_helpers::sources::source_deferral; +use pyo3::prelude::{Bound, PyResult}; +use pyo3::types::{PyDict, PyDict as KeywordDict}; + +/// Resolve source reads for selected SQL, or for `read_source_names` when given. +#[inline] +pub fn build_source_read_map<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + source_deferral::build_source_read_map(py, options) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist.rs new file mode 100644 index 0000000000..6ad31a35c8 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist.rs @@ -0,0 +1,20 @@ +//! Raise S405 when a selected resource reads an external source table that does not exist. + +use crate::planner_warehouse::_helpers::sources::source_tables; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Raise S405 when a selected resource reads an external source table that does not exist. +#[inline] +pub fn check_selected_source_tables_exist<'py>( + project: &Bound<'py, PyAny>, + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, +) -> PyResult<()> { + source_tables::check_selected_source_tables_exist( + project, + (adapter, connection), + scope, + relations, + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist_binding_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist_binding_function.rs new file mode 100644 index 0000000000..7c84f33321 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/check_selected_source_tables_exist_binding_function.rs @@ -0,0 +1,10 @@ +//! `check_selected_source_tables_exist_binding` as a function of `module`. + +use crate::planner_warehouse::_helpers::sources::source_tables::check_selected_source_tables_exist_binding; + +/// `check_selected_source_tables_exist_binding` as a function of `module`. +pub fn check_selected_source_tables_exist_binding_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(check_selected_source_tables_exist_binding, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns.rs new file mode 100644 index 0000000000..d3d36b5bb4 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns.rs @@ -0,0 +1,17 @@ +//! Gather source columns. + +use crate::planner_warehouse::_helpers::sources::source_columns::source_inspection; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyDict; + +#[pyfunction] +#[pyo3(signature = (*, project, adapter, connection, source_entries=None))] +pub fn gather_source_columns<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + source_entries: Option>, +) -> PyResult> { + Ok(source_inspection(project, (adapter, connection), source_entries)?.0) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns_function.rs new file mode 100644 index 0000000000..1088a18001 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_columns_function.rs @@ -0,0 +1,10 @@ +//! `gather_source_columns` as a function of `module`. + +use crate::planner_warehouse::main::sources::gather_source_columns::gather_source_columns; + +/// `gather_source_columns` as a function of `module`. +pub fn gather_source_columns_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(gather_source_columns, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_inspection.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_inspection.rs new file mode 100644 index 0000000000..62fe3bdf29 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/gather_source_inspection.rs @@ -0,0 +1,16 @@ +//! Gather source inspection. + +use crate::planner_warehouse::_helpers::sources::source_columns::source_inspection; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyTuple; + +/// Gather source inspection. +pub fn gather_source_inspection<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + source_entries: Option>, +) -> PyResult> { + let (columns, listed) = source_inspection(project, (adapter, connection), source_entries)?; + Ok(PyTuple::new(project.py(), [columns.into_any(), listed])?.into_any()) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/list_source_candidates_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/list_source_candidates_function.rs new file mode 100644 index 0000000000..b87ee0d23f --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/list_source_candidates_function.rs @@ -0,0 +1,10 @@ +//! `list_source_candidates` as a function of `module`. + +use crate::planner_warehouse::_helpers::sources::semantic_sources::list_source_candidates; + +/// `list_source_candidates` as a function of `module`. +pub fn list_source_candidates_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(list_source_candidates, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/mod.rs new file mode 100644 index 0000000000..2ec99c7e4a --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/mod.rs @@ -0,0 +1,13 @@ +pub mod build_planner_source_freshness_result; +pub mod build_planner_source_freshness_result_of_function; +pub mod build_source_read_map; +pub mod check_selected_source_tables_exist; +pub mod check_selected_source_tables_exist_binding_function; +pub mod gather_source_columns; +pub mod gather_source_columns_function; +pub mod gather_source_inspection; +pub mod list_source_candidates_function; +pub mod semantic_source_columns; +pub mod semantic_source_columns_function; +pub mod source_freshness_unknown_warnings; +pub mod with_declared_source_reads; diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns.rs new file mode 100644 index 0000000000..89a3594a63 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns.rs @@ -0,0 +1,14 @@ +//! Reuse collected columns and inspect only selected, unresolved table sources. + +use crate::planner_warehouse::_helpers::sources::semantic_sources; +use pyo3::prelude::{Bound, PyResult}; +use pyo3::types::{PyDict, PyDict as KeywordDict}; + +/// Reuse collected columns and inspect only selected, unresolved table sources. +#[inline] +pub fn semantic_source_columns<'py>( + py: pyo3::Python<'py>, + options: Option>, +) -> PyResult> { + semantic_sources::semantic_source_columns(py, options) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns_function.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns_function.rs new file mode 100644 index 0000000000..8d85cb0511 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/semantic_source_columns_function.rs @@ -0,0 +1,10 @@ +//! `semantic_source_columns` as a function of `module`. + +use crate::planner_warehouse::_helpers::sources::semantic_sources::semantic_source_columns; + +/// `semantic_source_columns` as a function of `module`. +pub fn semantic_source_columns_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(semantic_source_columns, module) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/source_freshness_unknown_warnings.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/source_freshness_unknown_warnings.rs new file mode 100644 index 0000000000..db67581c12 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/source_freshness_unknown_warnings.rs @@ -0,0 +1,54 @@ +//! One warning per reason that left configured source freshness unknown. + +use crate::freshness::constants::PREVIEW_MESSAGE; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::planner_warehouse::main::freshness_unknown_message::freshness_unknown_message; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; + +/// One warning per reason that left configured source freshness unknown. +pub fn source_freshness_unknown_warnings<'py>( + source_freshness: &Bound<'py, PyAny>, +) -> PyResult> { + let py = source_freshness.py(); + let mut reasons: Vec<(String, Vec)> = Vec::new(); + for unknown in native_field(source_freshness, "unknown_sources")? + .call_method0("values")? + .try_iter()? + { + let unknown = unknown?; + let reason: String = native_field(&unknown, "reason")?.str()?.extract()?; + if freshness_unknown_message(&reason, &[]).is_none() + || native_field(&unknown, "message")?.eq(PREVIEW_MESSAGE)? + { + continue; + } + let name: String = native_field(&unknown, "source_name")?.extract()?; + match reasons.iter_mut().find(|(existing, _)| *existing == reason) { + Some((_, names)) => names.push(name), + None => reasons.push((reason, vec![name])), + } + } + let mut warnings: Vec<_> = Vec::new(); + for (reason, names) in reasons { + if let Some(message) = freshness_unknown_message(&reason, &names) { + warnings.push(native( + py, + RecordId::PlanWarning, + &[ + ("model_name", py.None().into_bound(py)), + ( + "severity", + enum_member(py, RecordId::WarningSeverity, "WARNING")?, + ), + ("message", text(py, &message)), + ], + )?); + } + } + tuple(py, warnings) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/with_declared_source_reads.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/with_declared_source_reads.rs new file mode 100644 index 0000000000..a7f8edbd4c --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/with_declared_source_reads.rs @@ -0,0 +1,35 @@ +//! Add declared project sources that no selected SQL resource reads to a source map. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::project::sources::sources; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; +use std::collections::BTreeSet; + +/// Add declared project sources that no selected SQL resource reads to a source map. +pub fn with_declared_source_reads<'py>( + project: &Bound<'py, PyAny>, + source_map: &Bound<'py, PyDict>, + source_names: &Bound<'py, PyAny>, +) -> PyResult> { + let mut missing: BTreeSet<_> = BTreeSet::new(); + for name in source_names.try_iter()? { + let name: String = name?.extract()?; + if !source_map.contains(&name)? { + missing.insert(name); + } + } + if missing.is_empty() { + return Ok(source_map.clone()); + } + let result = source_map.copy()?; + for source in sources(project)?.try_iter()? { + let entry = source_entry(&source?)?; + let name: String = native_field(&entry, "name")?.extract()?; + if missing.contains(&name) { + result.set_item(name, entry)?; + } + } + Ok(result) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/mod.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/mod.rs new file mode 100644 index 0000000000..106e3b1428 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/mod.rs @@ -0,0 +1,6 @@ +//! Native warehouse inspection for planning over Python adapter callbacks. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs similarity index 67% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs rename to crates/sqlbuild-python-planner/src/planner_warehouse/models.rs index d5c1e961ed..fc106c2d09 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs @@ -1,137 +1,63 @@ -//! Migration fingerprints computed at most once per model definition and reused across runs. +//! Public records of planner warehouse. -use crate::bindings::_helpers::planner_changes::metadata::without_declared_columns_hash; -use crate::bindings::_helpers::planner_migrations::fingerprint::fingerprint; -use crate::bindings::_helpers::planner_migrations::fingerprint::migration_fingerprint_ref_names; -use crate::bindings::_helpers::planner_warehouse::models::FingerprintWork; -use pyo3::PyRefMut; -use pyo3::exceptions::{PyOSError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyBool, PyDict, PyFloat, PyList, PyListMethods, PyTuple}; -use pyo3::{pyclass, pyfunction, pymethods}; +use crate::planner_changes::main::without_declared_columns_hash::without_declared_columns_hash; +use crate::planner_migrations::main::fingerprint::fingerprint; +use crate::planner_migrations::main::migration_fingerprint_ref_names::migration_fingerprint_ref_names; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::CACHE_DATABASE_NAME; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::CACHE_DIRECTORY_NAME; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::CREATE_CACHE_TABLE_SQL; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::INSERT_SQL; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::MemoKey; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::SELECT_SQL; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::close; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::connect; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::count_computation; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::package_version; +use crate::planner_warehouse::_helpers::inspection::fingerprint_cache::sqlite_error; +use pyo3::exceptions::PyOSError; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyBool; +use pyo3::types::PyDict; +use pyo3::types::PyList; +use pyo3::types::PyListMethods; +use pyo3::types::PyTuple; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_entry::fingerprint_cache_entry; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_key::fingerprint_cache_key; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_lookup::fingerprint_cache_lookup; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_shared_digest::fingerprint_cache_shared_digest; use sqlbuild_analysis::planner_warehouse::models::FingerprintCacheKey; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_project::planner_phases::main::object::object; -use std::collections::{BTreeMap, HashMap}; -use std::hash::{DefaultHasher, Hash, Hasher}; - -const CACHE_DIRECTORY_NAME: &str = "migration-fingerprints-v1"; -const CACHE_DATABASE_NAME: &str = "migration-fingerprints.sqlite3"; -const SQLITE_TIMEOUT_SECONDS: f64 = 0.1; -const CREATE_CACHE_TABLE_SQL: &str = "\nCREATE TABLE IF NOT EXISTS migration_fingerprint (\n cache_key TEXT PRIMARY KEY,\n payload TEXT NOT NULL\n)\n"; -const INSERT_SQL: &str = - "INSERT OR REPLACE INTO migration_fingerprint (cache_key, payload) VALUES (?, ?)"; -const SELECT_SQL: &str = "SELECT payload FROM migration_fingerprint WHERE cache_key = ?"; - -/// A fingerprint request: query SQL, metadata JSON, dialect and visible renames. -#[derive(Hash, PartialEq, Eq)] -struct MemoKey { - query_sql: String, - metadata_json: String, - dialect: Option, - visible_renames: Vec<(String, String)>, -} - -static FINGERPRINT_WORK: PyOnceLock> = PyOnceLock::new(); - -fn fingerprint_work(py: Python<'_>) -> PyResult> { - Ok(FINGERPRINT_WORK - .get_or_try_init(py, || Py::new(py, FingerprintWork::default()))? - .bind(py) - .borrow_mut()) -} - -fn count_computation(py: Python<'_>, key: &MemoKey) -> PyResult<()> { - let mut hasher = DefaultHasher::new(); - key.hash(&mut hasher); - let mut work = fingerprint_work(py)?; - work.computations += 1; - work.distinct.insert(hasher.finish()); - Ok(()) -} - -/// Report migration fingerprints computed and distinct requests among them, optionally resetting. -#[pyfunction] -#[pyo3(signature = (*, reset = false))] -pub(crate) fn migration_fingerprint_work_info( - py: Python<'_>, - reset: bool, -) -> PyResult<(u64, usize)> { - let mut work = fingerprint_work(py)?; - let info = (work.computations, work.distinct.len()); - if reset { - *work = FingerprintWork::default(); - } - Ok(info) -} +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::collections::HashMap; /// Reuse migration fingerprints across rename maps, planning phases and invocations. #[pyclass(module = "sqlbuild._native")] -pub(crate) struct MigrationFingerprintCache { - ref_names: HashMap<(String, String), Vec>, - fingerprints: HashMap>, - database_path: Option>, - connection: Option>, - connection_opened: bool, - pending: Vec<(String, String)>, - shared_digest: String, -} - -fn sqlite_error(py: Python<'_>, error: &PyErr) -> PyResult { - Ok(error.is_instance(py, &imported(py, "sqlite3", "DatabaseError")?)) -} - -fn close(py: Python<'_>, connection: Option>) -> PyResult<()> { - let Some(connection) = connection else { - return Ok(()); - }; - match connection.bind(py).call_method0("close") { - Ok(_) => Ok(()), - Err(error) if sqlite_error(py, &error)? => Ok(()), - Err(error) => Err(error), - } -} - -fn connect<'py>(path: &Bound<'py, PyAny>) -> PyResult> { - let py = path.py(); - call_with( - &imported(py, "sqlite3", "connect")?, - &[ - ("database", path.clone()), - ( - "timeout", - PyFloat::new(py, SQLITE_TIMEOUT_SECONDS).into_any(), - ), - ], - ) -} - -fn package_version(py: Python<'_>, package: &str) -> PyResult { - match imported(py, "importlib.metadata", "version")?.call1((package,)) { - Ok(version) => version.extract(), - Err(error) - if error.is_instance( - py, - &imported(py, "importlib.metadata", "PackageNotFoundError")?, - ) => - { - Ok("unknown".to_owned()) - } - Err(error) => Err(error), - } +#[derive(Debug)] +pub struct MigrationFingerprintCache { + pub(crate) ref_names: HashMap<(String, String), Vec>, + pub(crate) fingerprints: HashMap>, + pub(crate) database_path: Option>, + pub(crate) connection: Option>, + pub(crate) connection_opened: bool, + pub(crate) pending: Vec<(String, String)>, + pub(crate) shared_digest: String, } impl MigrationFingerprintCache { /// A cache persisting under `root`, or an in-memory cache without one. - pub(crate) fn open(py: Python<'_>, root: Option>) -> PyResult { + pub fn open(py: Python<'_>, root: Option>) -> PyResult { let root = root.filter(|root| !root.is_none()); let database_path = match &root { Some(root) => Some( @@ -161,7 +87,7 @@ impl MigrationFingerprintCache { }) } - fn cache_key(&self, key: &MemoKey) -> PyResult { + pub(crate) fn cache_key(&self, key: &MemoKey) -> PyResult { if self.database_path.is_none() { return Ok(String::new()); } @@ -175,7 +101,7 @@ impl MigrationFingerprintCache { .map_err(|error| PyValueError::new_err(format!("{error:?}"))) } - fn open_connection(&mut self, py: Python<'_>) -> PyResult>> { + pub(crate) fn open_connection(&mut self, py: Python<'_>) -> PyResult>> { if self.connection_opened { return Ok(self .connection @@ -203,7 +129,11 @@ impl MigrationFingerprintCache { .map(|connection| connection.clone_ref(py))) } - fn stored(&mut self, py: Python<'_>, cache_key: &str) -> PyResult>> { + pub(crate) fn stored( + &mut self, + py: Python<'_>, + cache_key: &str, + ) -> PyResult>> { let Some(connection) = self.open_connection(py)? else { return Ok(None); }; @@ -230,7 +160,7 @@ impl MigrationFingerprintCache { Ok(fingerprint_cache_lookup(&contents, cache_key)) } - fn remember(&mut self, cache_key: &str, fingerprint: Option<&str>) -> PyResult<()> { + pub(crate) fn remember(&mut self, cache_key: &str, fingerprint: Option<&str>) -> PyResult<()> { if self.database_path.is_none() { return Ok(()); } @@ -243,7 +173,7 @@ impl MigrationFingerprintCache { Ok(()) } - fn write_pending( + pub(crate) fn write_pending( &self, py: Python<'_>, pending: &[(String, String)], @@ -297,13 +227,13 @@ impl MigrationFingerprintCache { impl MigrationFingerprintCache { #[new] #[pyo3(signature = (*, root=None))] - fn new(py: Python<'_>, root: Option>) -> PyResult { + pub(crate) fn new(py: Python<'_>, root: Option>) -> PyResult { Self::open(py, root) } /// The migration fingerprint, keyed only by the renames this model can see. #[pyo3(signature = (**options))] - fn fingerprint( + pub(crate) fn fingerprint( &mut self, py: Python<'_>, options: Option>, @@ -377,7 +307,7 @@ impl MigrationFingerprintCache { } /// Write fingerprints computed since the last persist; failures only skip reuse. - fn persist(&mut self, py: Python<'_>) -> PyResult<()> { + pub(crate) fn persist(&mut self, py: Python<'_>) -> PyResult<()> { let pending = std::mem::take(&mut self.pending); let connection = self.connection.take(); self.connection_opened = false; @@ -391,3 +321,26 @@ impl MigrationFingerprintCache { closed } } + +/// Migration fingerprints computed since the last reset, for work-count tests. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct FingerprintWork { + pub computations: u64, + pub distinct: BTreeSet, +} + +/// Everything one snapshot gather needs. +#[derive(Debug)] +pub struct SnapshotRequest<'py> { + pub project: Bound<'py, PyAny>, + pub adapter: Bound<'py, PyAny>, + pub connection: Bound<'py, PyAny>, + pub execute: Bound<'py, PyAny>, + pub selected_keys: Bound<'py, PyAny>, + pub full_refresh: bool, + pub full_refresh_model_names: Bound<'py, PyAny>, + pub on_progress: Bound<'py, PyAny>, + pub deferred_locations: Bound<'py, PyAny>, + pub cursor_scope: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs index bd56463a03..06ac1d20c1 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs @@ -10,8 +10,11 @@ use std::sync::Arc; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use crate::native_host::models::ModelHandle; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_base::records::models::NativeRecord; +use sqlbuild_python_base::records::types::RecordId; /// Read native ownership directly, or import a caller-authored public scope at the API edge. pub(crate) fn scope_data(value: &Bound<'_, PyAny>) -> PyResult> { @@ -54,3 +57,38 @@ pub(crate) fn scope_data(value: &Bound<'_, PyAny>) -> PyResult> { .collect(), })) } + +/// Preserve public record identity and dataclass behavior on explicit projection. +pub(crate) fn project_scope<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let Ok(scope) = value.cast::() else { + return Ok(value.clone()); + }; + scope + .get() + .projection + .get_or_try_init(value.py(), || { + let names = [ + "upstream_deps", + "downstream_deps", + "all_keys", + "models_by_name", + "selected_keys", + "execution_order", + "user_selected_keys", + "python_read_source_names", + "sql_test_selection", + "load_sources", + "end_audit_source_keys", + ]; + let fields = names + .iter() + .map(|name| Ok((*name, scope.get().data.field(value.py(), name)?))) + .collect::>>()?; + Ok::<_, pyo3::PyErr>( + new_record_with(value.py(), RecordId::PlannerScope, &fields)? + .cast_into::() + .map(Bound::unbind)?, + ) + }) + .map(|value| value.bind(scope.py()).clone().into_any()) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/mod.rs b/crates/sqlbuild-python-project/src/planner_scope/main/mod.rs index 03a88c4a60..84294e6dc1 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/main/mod.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/mod.rs @@ -9,7 +9,8 @@ pub mod edges; pub mod expand_selected_loader_dependencies_function; pub mod keys; pub mod project_graph; -pub mod project_scope; +pub mod resolve_planner_scopes; +pub mod resolve_planner_scopes_function; pub mod scope_data; pub mod source_load_map; pub mod upstream_loader_dependency_names; diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs b/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs deleted file mode 100644 index 734d1789c4..0000000000 --- a/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs +++ /dev/null @@ -1,42 +0,0 @@ -//! Preserve public record identity and dataclass behavior on explicit projection. - -use crate::planner_scope::models::RetainedScope; -use pyo3::prelude::{Bound, PyAny, PyResult}; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::models::NativeRecord; -use sqlbuild_python_base::records::types::RecordId; - -/// Preserve public record identity and dataclass behavior on explicit projection. -pub fn project_scope<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - let Ok(scope) = value.cast::() else { - return Ok(value.clone()); - }; - scope - .get() - .projection - .get_or_try_init(value.py(), || { - let names = [ - "upstream_deps", - "downstream_deps", - "all_keys", - "models_by_name", - "selected_keys", - "execution_order", - "user_selected_keys", - "python_read_source_names", - "sql_test_selection", - "load_sources", - "end_audit_source_keys", - ]; - let fields = names - .iter() - .map(|name| Ok((*name, scope.get().data.field(value.py(), name)?))) - .collect::>>()?; - Ok::<_, pyo3::PyErr>( - new_record_with(value.py(), RecordId::PlannerScope, &fields)? - .cast_into::() - .map(Bound::unbind)?, - ) - }) - .map(|value| value.bind(scope.py()).clone().into_any()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs b/crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs rename to crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes.rs index 6e72967165..1c26f9b08c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes.rs @@ -1,18 +1,18 @@ //! Resolve the selected, stale-warning and inspection scopes of one plan. -use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; +use crate::planner_scope::main::build_planner_scope::build_planner_scope; +use crate::planner_scope::main::scope_data::scope_data; +use crate::planner_scope::models::PlannerScopeResolution; use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::pyfunction; use pyo3::types::PyTuple; use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_project::planner_scope::main::build_planner_scope::build_planner_scope; -use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; #[pyfunction] #[pyo3(signature = (*, project, selection, policies))] -pub(crate) fn resolve_planner_scopes<'py>( +pub fn resolve_planner_scopes<'py>( project: &Bound<'py, PyAny>, selection: &Bound<'py, PyAny>, policies: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes_function.rs new file mode 100644 index 0000000000..3df9aa64de --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/resolve_planner_scopes_function.rs @@ -0,0 +1,10 @@ +//! `resolve_planner_scopes` as a function of `module`. + +use crate::planner_scope::main::resolve_planner_scopes::resolve_planner_scopes; + +/// `resolve_planner_scopes` as a function of `module`. +pub fn resolve_planner_scopes_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(resolve_planner_scopes, module) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/models.rs b/crates/sqlbuild-python-project/src/planner_scope/models.rs index 91a26242c2..e807f3c8f9 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/models.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/models.rs @@ -3,6 +3,7 @@ use crate::graph::main::project::edge_dict::edge_dict; use crate::graph::models::KeyCache; use crate::native_host::models::ModelHandle; +use crate::planner_scope::_helpers::retained::project_scope; use pyo3::IntoPyObjectExt; use pyo3::prelude::Bound; use pyo3::prelude::Py; @@ -10,6 +11,7 @@ use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; use pyo3::prelude::Python; use pyo3::pyclass; +use pyo3::pymethods; use pyo3::sync::PyOnceLock; use pyo3::types::PyDict; use pyo3::types::PyDictMethods; @@ -129,3 +131,44 @@ impl ScopeData { } } } + +/// Resolved planner scopes for one plan build. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct PlannerScopeResolution { + pub selected_scope: Py, + pub stale_warning_scope: Py, + pub inspection_scope: Py, +} + +#[pymethods] +impl PlannerScopeResolution { + #[new] + #[pyo3(signature = (*, selected_scope, stale_warning_scope, inspection_scope))] + fn new( + selected_scope: Py, + stale_warning_scope: Py, + inspection_scope: Py, + ) -> Self { + Self { + selected_scope, + stale_warning_scope, + inspection_scope, + } + } + + #[getter] + fn selected_scope<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { + project_scope(self.selected_scope.bind(py)) + } + + #[getter] + fn stale_warning_scope<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { + project_scope(self.stale_warning_scope.bind(py)) + } + + #[getter] + fn inspection_scope<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { + project_scope(self.inspection_scope.bind(py)) + } +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 75424c5112..7e7f354170 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -31,7 +31,6 @@ sqlbuild-cache.workspace = true sqlbuild-cli.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true -sqlbuild-cursor.workspace = true sqlbuild-discovery.workspace = true sqlbuild-host.workspace = true sqlbuild-model-config.workspace = true @@ -41,6 +40,7 @@ sqlbuild-pysource.workspace = true sqlbuild-python-base.workspace = true sqlbuild-python-front.workspace = true sqlbuild-python-project.workspace = true +sqlbuild-python-planner.workspace = true sqlbuild-recall.workspace = true sqlbuild-refactor.workspace = true sqlbuild-render.workspace = true @@ -49,7 +49,6 @@ sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlbuild-state.workspace = true sqlclean.workspace = true -uuid.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 8eb7797022..f7224beb48 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -22,9 +22,7 @@ use crate::bindings::_helpers::build_progress::progress_text::write_execution_he use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; -use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::planner_entries::command_plans::build_audit_command_plan; use crate::bindings::_helpers::result_output::documents::{ audit_execution_json, audit_projection, write_execution_json_output, }; @@ -42,6 +40,8 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_front::cost_context::models::CostContext; +use sqlbuild_python_planner::planner_entries::main::plans::build_audit_command_plan::build_audit_command_plan; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; use sqlbuild_python_project::project_snapshot::types::VerifiedRun; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 08bb455cde..92d599550d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -22,9 +22,6 @@ use crate::bindings::_helpers::check::check_core::relevant_check_functions; use crate::bindings::_helpers::compile_driver::targets::{ TargetSettingsValues, selected_target_settings, }; -use crate::bindings::_helpers::observations::source_capture::{ - arm_source_capture, take_source_capture, -}; use crate::bindings::_helpers::recall::console::write_stderr; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; @@ -38,6 +35,8 @@ use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_use use sqlbuild_python_front::discovery::main::sources::provider_uses::provider_uses; use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; +use sqlbuild_python_planner::observations::main::arm_source_capture::arm_source_capture; +use sqlbuild_python_planner::observations::main::take_source_capture::take_source_capture; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index c633991b0a..3eb750f15f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -28,14 +28,6 @@ use crate::bindings::_helpers::check::check_results::write_check_results; use crate::bindings::_helpers::command_runtime::warehouses::describe_connection_warehouse; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; use crate::bindings::_helpers::compile_driver::targets::project_target_settings; -use crate::bindings::_helpers::consumption::policy::{changed_policy, policy_line}; -use crate::bindings::_helpers::consumption::recorder::{ - ConsumptionRecorder, Prepared, held_back_text, prepare as prepare_consumption, run_reason_text, -}; -use crate::bindings::_helpers::cost_telemetry::context::CostContext; -use crate::bindings::_helpers::freshness::publication::append_freshness_records; -use crate::bindings::_helpers::observations::publication::append_build_observations; -use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::sql_test_command::test_command::timeout_record; use crate::bindings::_helpers::sql_test_command::timeouts::{TimeoutKind, resolve_run_timeout}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -57,9 +49,19 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cost_context::models::CostContext; use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; use sqlbuild_python_front::native_host::main::recorder_callbacks::recorder_callbacks; +use sqlbuild_python_planner::consumption::main::changed_policy::changed_policy; +use sqlbuild_python_planner::consumption::main::held_back_text::held_back_text; +use sqlbuild_python_planner::consumption::main::policy_line::policy_line; +use sqlbuild_python_planner::consumption::main::prepare::prepare as prepare_consumption; +use sqlbuild_python_planner::consumption::main::run_reason_text::run_reason_text; +use sqlbuild_python_planner::consumption::models::{ConsumptionRecorder, Prepared}; +use sqlbuild_python_planner::freshness::main::append_freshness_records::append_freshness_records; +use sqlbuild_python_planner::observations::main::append_build_observations::append_build_observations; +use sqlbuild_python_planner::planner_resolution::main::cursor::type_cursor_overrides::type_cursor_overrides; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_state::consumption::models::ChangedPolicy; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index fb29fddc1f..462324401a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -15,7 +15,6 @@ use crate::bindings::_helpers::build_progress::progress_text::with_verbose_inspe use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::manifest::builder::build_manifest; -use crate::bindings::_helpers::plan_output::plan_text::format_plan; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::build_command::main::boolean::boolean; @@ -27,6 +26,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_planner::plan_output::main::format_plan::format_plan; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs index 625fb38bf1..915ef8e1f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs @@ -10,8 +10,8 @@ use sqlbuild_core::plan_display::main::model_resource_type::model_resource_type; use sqlbuild_core::plan_display::models::ModelLabelFacts; use crate::bindings::_helpers::build_progress::result_text::text; -use crate::bindings::_helpers::plan_output::display::optional_facts; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_planner::plan_output::main::optional_facts::optional_facts; const SOURCE_TYPE: &str = "source"; const SQL_TEST_TYPE: &str = "sql_test"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs index 6009f9bdc0..f5b61e58f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs @@ -25,10 +25,11 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::plan_output::display::{cursor_bound_display, inclusive_cursor_end}; use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::result_output::main::json_value::json_value; +use sqlbuild_python_planner::plan_output::main::cursor_bound_display::cursor_bound_display; +use sqlbuild_python_planner::plan_output::main::inclusive_cursor_end::inclusive_cursor_end; const DECIMAL_KIND: &str = "decimal"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs index 7cf4d4902f..f44cf2dc69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs @@ -21,8 +21,9 @@ use sqlbuild_analysis::definition_changes::models::{DefinitionSnapshot, NodeDefi use crate::bindings::_helpers::change_selection::fingerprints::{ Fingerprinting, directory_digest, file_digest, text_of, }; -use crate::bindings::_helpers::planner_identity::metadata::{function_local_hashes, seed_identity}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_planner::planner_identity::main::function_local_hashes::function_local_hashes; +use sqlbuild_python_planner::planner_identity::main::seed_identity::seed_identity; use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs index 9d570e4062..ea837282d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs @@ -26,9 +26,9 @@ use crate::bindings::_helpers::change_selection::definition_snapshot::{ declaration_uses, definition_snapshot, project_wide_keys, }; use crate::bindings::_helpers::change_selection::environment_nodes::with_current_dependencies; -use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; use crate::bindings::_helpers::scenario_command::setup::{flush, write}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_planner::planner_scenario::main::plan_scenario_graph::plan_scenario_graph; use sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index cb3d90f5f7..8f084c260f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -11,7 +11,6 @@ use crate::bindings::_helpers::compile_driver::targets::{ TargetSettingsValues, project_target_settings, }; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::planner_entries::command_plans::build_relation_command_plan; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -33,6 +32,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; +use sqlbuild_python_planner::planner_entries::main::plans::build_relation_command_plan::build_relation_command_plan; use sqlbuild_python_project::graph::main::nodes::sql_ref_key::sql_ref_key; use sqlbuild_python_project::graph::main::nodes::transitive_closure_many::transitive_closure_many; use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index 4fd56ca394..4809d49ef6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -28,7 +28,6 @@ use crate::bindings::_helpers::clone::clone_progress::{clone_item_text, write_li use crate::bindings::_helpers::command_runtime::connection::resolve_target_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::planner_entries::clone_plans::PhaseTimings; use crate::bindings::_helpers::result_output::documents::format_clone_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -47,6 +46,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; +use sqlbuild_python_planner::planner_entries::types::PhaseTimings; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; const CLONE_LOGGER: &str = "sqlbuild.clone"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs index 184caf11c2..067fdb3917 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs @@ -5,13 +5,14 @@ use pyo3::types::{PyBool, PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::planner_entries::clone_plans::{PhaseTimings, timed_clone_planning}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_planner::planner_entries::main::plans::timed_clone_planning::timed_clone_planning; +use sqlbuild_python_planner::planner_entries::types::PhaseTimings; /// The phase name of one target compile, before the target name. const COMPILE_PHASE_PREFIX: &str = "compile_"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 7cdd47a44b..22c26a8fa0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -9,10 +9,10 @@ use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_g use crate::bindings::_helpers::compile_driver::python_plan_entries::{ build_python_run_plan_outputs, python_read_source_names_for, }; -use crate::bindings::_helpers::planner_entries::execution_plan::build_execution_plan; use sqlbuild_analysis::planner_scope::constants::SOURCE; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_planner::planner_entries::main::plans::build_execution_plan::build_execution_plan; use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::graph::models::KeyCache; use sqlbuild_python_project::planner_phases::main::object::object; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index 005212247d..1ec02765cb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -8,13 +8,13 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::assembly::assemble_project; use crate::bindings::_helpers::compile_driver::pipeline_entries::compile_analysis_selection; -use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_scope::main::resolve_planner_scopes::resolve_planner_scopes; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use sqlbuild_python_project::planner_scope::models::ScopeData; use sqlbuild_python_project::planner_selection::main::split_project_unit_test_selectors::split_project_unit_test_selectors; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs index 12cc3c0e8e..446d4abf23 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs @@ -10,14 +10,13 @@ use crate::bindings::_helpers::compile_driver::deferral::{ use crate::bindings::_helpers::compile_driver::pipeline::compile_project_phase; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; use crate::bindings::_helpers::compile_driver::selection_phases::unit_test_selectors_applied; -use crate::bindings::_helpers::planner_entries::command_plans::{ - resolve_static_command_scope, resolve_static_relation_context, -}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_planner::planner_entries::main::plans::resolve_static_command_scope::resolve_static_command_scope; +use sqlbuild_python_planner::planner_entries::main::plans::resolve_static_relation_context::resolve_static_relation_context; use sqlbuild_python_project::graph::main::python_nodes::run_selection_from_inputs::run_selection_from_inputs; /// Locations of the deferred target's relations, when the options defer to one. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs deleted file mode 100644 index 299520c88f..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs +++ /dev/null @@ -1,129 +0,0 @@ -//! What `--changed` counts and how far a change reaches: the command line, else `[changes]`. - -use std::collections::BTreeSet; - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; -use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_state::consumption::models::{ - ChangeKind, ChangedPolicy, NodeKey, Reach, TriggeredJob, -}; - -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; -use sqlbuild_python_project::compiled_project::main::triggered_jobs::triggered_jobs; -use sqlbuild_python_project::graph::models::NativeProjectGraph; - -const DEFINITION: &str = "definition"; -const SOURCE_DATA: &str = "source_data"; -const REACH_JOB: &str = "job"; -const REACH_ALL: &str = "all"; - -fn change_kind(name: &str) -> PyResult { - match name { - DEFINITION => Ok(ChangeKind::Definition), - SOURCE_DATA => Ok(ChangeKind::SourceData), - other => Err(PyValueError::new_err(format!( - "unknown change kind '{other}': expected {DEFINITION} or {SOURCE_DATA}" - ))), - } -} - -/// A change kind as `[changes] counts` and `--changed=` spell it. -pub(crate) const fn kind_value(kind: ChangeKind) -> &'static str { - match kind { - ChangeKind::Definition => DEFINITION, - ChangeKind::SourceData => SOURCE_DATA, - } -} - -/// The request's `--select` selectors. -fn selectors(request: &Bound<'_, PyAny>) -> PyResult> { - Ok(native_field(request, "select")? - .extract::>>()? - .unwrap_or_default()) -} - -/// A triggered job with its selected nodes as consumption keys. -fn triggered_job((name, keys): (String, BTreeSet)) -> TriggeredJob { - TriggeredJob { - name, - nodes: keys - .into_iter() - .map(|(kind, name)| NodeKey { kind, name }) - .collect(), - } -} - -/// A reach as `[changes] reach` and `--reach` spell it. -pub(crate) const fn reach_value(reach: &Reach) -> &'static str { - match reach { - Reach::Job(_) => REACH_JOB, - Reach::All => REACH_ALL, - } -} - -/// The policy of a `--changed` request, or None without it; triggered jobs resolve on `project`. -pub(crate) fn changed_policy( - project_dir: &Bound<'_, PyAny>, - request: &Bound<'_, PyAny>, - project: &Bound<'_, PyAny>, -) -> PyResult> { - if !native_field(request, "changed")?.is_truthy()? { - return Ok(None); - } - let configured = configured_changes(project_dir)?; - let counts: Vec = match native_field(request, "changed_counts")?.extract()? { - Some(counts) => counts, - None => configured - .counts - .unwrap_or_else(|| vec![DEFINITION.to_owned()]), - }; - let reach: String = match native_field(request, "reach")?.extract()? { - Some(reach) => reach, - None => configured.reach.unwrap_or_else(|| REACH_JOB.to_owned()), - }; - let counts: BTreeSet = counts - .iter() - .map(|name| change_kind(name)) - .collect::>()?; - let reach: Reach = match reach.as_str() { - REACH_ALL => Reach::All, - REACH_JOB => { - let graph: PyRef<'_, NativeProjectGraph> = - native_field(project, "lineage_graph")?.extract()?; - Reach::Job( - triggered_jobs(project.py(), graph.graph(), &selectors(request)?)? - .into_iter() - .map(triggered_job) - .collect(), - ) - } - other => { - return Err(PyValueError::new_err(format!( - "unknown reach '{other}': expected {REACH_JOB} or {REACH_ALL}" - ))); - } - }; - Ok(Some(ChangedPolicy { counts, reach })) -} - -/// One line saying what the policy counts and how far it reaches, as build and plan print it. -pub(crate) fn policy_line(policy: &ChangedPolicy) -> String { - let label = |kind: &ChangeKind| match kind { - ChangeKind::Definition => "definition", - ChangeKind::SourceData => "source data", - }; - let counted: Vec<&str> = policy.counts.iter().map(label).collect(); - let uncounted: Vec = [ChangeKind::Definition, ChangeKind::SourceData] - .iter() - .filter(|kind| !policy.counts.contains(kind)) - .map(|kind| format!(" ({} not counted)", label(kind))) - .collect(); - format!( - "Counting: {}{} ยท reach: {}", - counted.join(", "), - uncounted.concat(), - reach_value(&policy.reach) - ) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs deleted file mode 100644 index ecf1b8c59c..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs +++ /dev/null @@ -1,502 +0,0 @@ -//! Record what each model and seed execution consumed, and decide `--changed` skips, around one build. - -use std::collections::{BTreeMap, BTreeSet, HashMap}; -use std::sync::Mutex; -use std::time::{Duration, Instant}; - -use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::{pyclass, pymethods}; -use sqlbuild_state::consumption::constants::RECENT_EXECUTIONS; -use sqlbuild_state::consumption::main::changed_run_set::changed_run_set; -use sqlbuild_state::consumption::main::consumed_inputs::consumed_inputs; -use sqlbuild_state::consumption::main::consumption_history::consumption_history; -use sqlbuild_state::consumption::main::read_back_history::read_back_history; -use sqlbuild_state::consumption::main::started_append::started_append; -use sqlbuild_state::consumption::main::succeeded_appends::succeeded_appends; -use sqlbuild_state::consumption::models::{ - ChangedOutcome, ChangedPolicy, ConsumptionAppend, ConsumptionHistory, ConsumptionRow, - ForcedRun, HeldBack, InputFact, NodeFacts, NodeKey, Reach, RelationKey, RunReason, -}; -use uuid::Uuid; - -use crate::bindings::_helpers::consumption::facts::{ - BuildFacts, MODEL, SEED, build_facts, enum_text, -}; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::native_host::models::{ - ConsumptionConnection, Location, NodeCompleteHook, -}; - -/// Succeeded rows are flushed once this many executions are pending. -const FLUSH_EXECUTIONS: usize = 100; -/// Or once this long has passed since the last flush. -const FLUSH_INTERVAL: Duration = Duration::from_secs(30); -const MODEL_RESULT: &str = "model_name"; -const SEED_RESULT: &str = "seed_name"; -const SUCCESS: &str = "success"; -const ERROR_OUTCOME: &str = "error"; - -/// The mutable side of one build's recording. -struct Recording { - pending: BTreeMap>, - pending_executions: usize, - pending_seeds: Vec, - tables: BTreeSet, - last_flush: Instant, - warnings: Vec, -} - -impl Default for Recording { - fn default() -> Self { - Self { - pending: BTreeMap::new(), - pending_executions: 0, - pending_seeds: Vec::new(), - tables: BTreeSet::new(), - last_flush: Instant::now(), - warnings: Vec::new(), - } - } -} - -/// Appends `started` rows before a build and `succeeded` rows as its models and seeds complete. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct ConsumptionRecorder { - connection: ConsumptionConnection, - run_id: String, - nodes: HashMap, - history: ConsumptionHistory, - executions: HashMap, - failed_start: HashMap, - base_on_node_complete: NodeCompleteHook, - recording: Mutex, -} - -/// What `--changed` decided, for the build's report. -pub(crate) struct ChangedDecision { - pub(crate) policy: ChangedPolicy, - pub(crate) run: usize, - /// Unchanged nodes, which are up to date. - pub(crate) skipped: Vec, - /// Nodes downstream of a change that wait for a later run. - pub(crate) held_back: Vec<(NodeKey, HeldBack)>, - pub(crate) reasons: Vec<(NodeKey, RunReason)>, -} - -impl ChangedDecision { - /// Every selected node `--changed` leaves unbuilt: unchanged and held back. - pub(crate) fn not_run(&self) -> impl Iterator { - self.skipped - .iter() - .chain(self.held_back.iter().map(|(key, _)| key)) - } -} - -fn kind_of(result: &Bound<'_, PyAny>) -> PyResult> { - for (field, kind) in [(MODEL_RESULT, MODEL), (SEED_RESULT, SEED)] { - if let Some(name) = result.getattr_opt(field)? { - return Ok(Some((kind, name.extract()?))); - } - } - Ok(None) -} - -impl ConsumptionRecorder { - fn lock(&self) -> PyResult> { - self.recording - .lock() - .map_err(|_| PyRuntimeError::new_err("consumption recorder state is poisoned")) - } - - /// The succeeded rows of one execution that wrote its relation, with their schema. - fn succeeded(&self, key: &NodeKey) -> Option<(Location, Vec)> { - let (Some(node), Some(execution_id)) = (self.nodes.get(key), self.executions.get(key)) - else { - return None; - }; - let inputs = consumed_inputs(node, (&self.nodes, &self.history), &self.executions); - Some(( - Location::of(&node.relation), - succeeded_appends(node, (execution_id, &self.run_id), &inputs), - )) - } - - /// The recording with `key`'s succeeded rows queued. - fn queued(&self, mut recording: Recording, key: &NodeKey) -> Recording { - if let Some((location, rows)) = self.succeeded(key) { - recording.pending.entry(location).or_default().extend(rows); - recording.pending_executions += 1; - } - recording - } - - /// The recording after appending every pending row; a failure is reported, and its executions run again. - fn flushed(&self, py: Python<'_>, mut recording: Recording) -> Recording { - let pending = std::mem::take(&mut recording.pending); - let wrote: bool = !pending.is_empty(); - for (location, rows) in pending { - let created: bool = !recording.tables.insert(location.clone()); - let written: PyResult<()> = (if created { - Ok(()) - } else { - self.connection.ensure_table(py, &location) - }) - .and_then(|()| self.connection.append(py, &location, &rows)); - if let Err(error) = written { - recording.warnings.push(format!( - "could not record consumed inputs in schema {}: {error}; those models and seeds run again under --changed", - location.schema - )); - } - } - if wrote && let Err(error) = self.connection.close(py) { - recording.warnings.push(format!( - "could not close the consumed-input connection: {error}" - )); - } - recording.pending_executions = 0; - recording.last_flush = Instant::now(); - recording - } - - /// Queue the seeds whose error audits passed and flush everything. - pub(crate) fn finish( - &self, - py: Python<'_>, - build_result: &Bound<'_, PyAny>, - ) -> PyResult> { - let mut failed_audits: BTreeSet = BTreeSet::new(); - for audit in native_field(build_result, "source_audit_results")?.try_iter()? { - let audit: Bound<'_, PyAny> = audit?; - let target: Bound<'_, PyAny> = native_field(&audit, "attached_target_name")?; - if !target.is_none() - && enum_text(&native_field(&audit, "outcome")?)?.eq_ignore_ascii_case(ERROR_OUTCOME) - { - failed_audits.insert(target.extract()?); - } - } - let mut recording = self.lock()?; - for seed in std::mem::take(&mut recording.pending_seeds) { - if !failed_audits.contains(&seed.name) { - *recording = self.queued(std::mem::take(&mut *recording), &seed); - } - } - *recording = self.flushed(py, std::mem::take(&mut *recording)); - Ok(std::mem::take(&mut recording.warnings)) - } -} - -#[pymethods] -impl ConsumptionRecorder { - /// Forward a finished node, then queue a successful model, or a seed until its audits are known. - fn on_node_complete(&self, py: Python<'_>, node_result: Bound<'_, PyAny>) -> PyResult<()> { - self.base_on_node_complete.forward(&node_result)?; - let Some((kind, name)) = kind_of(&node_result)? else { - return Ok(()); - }; - if !enum_text(&native_field(&node_result, "status")?)?.eq_ignore_ascii_case(SUCCESS) { - return Ok(()); - } - let key = NodeKey { - kind: kind.to_owned(), - name, - }; - let mut recording = self.lock()?; - if kind == SEED { - recording.pending_seeds.push(key); - return Ok(()); - } - *recording = self.queued(std::mem::take(&mut *recording), &key); - if recording.pending_executions >= FLUSH_EXECUTIONS - || recording.last_flush.elapsed() >= FLUSH_INTERVAL - { - *recording = self.flushed(py, std::mem::take(&mut *recording)); - } - Ok(()) - } - - /// Fail a model or seed whose `started` row could not be appended, before it writes. - #[pyo3(signature = (*, entry, connection))] - fn before_node_write( - &self, - entry: Bound<'_, PyAny>, - connection: Bound<'_, PyAny>, - ) -> PyResult<()> { - let _ = connection; - let name: String = native_field(&entry, "name")?.extract()?; - let kind: &str = if entry.hasattr("csv_settings")? { - SEED - } else { - MODEL - }; - let key = NodeKey { - kind: kind.to_owned(), - name, - }; - match self.failed_start.get(&key) { - Some(error) => Err(PyRuntimeError::new_err(format!( - "not written: SQLBuild could not record that this {kind} started ({error}), so a crash could not be detected later" - ))), - None => Ok(()), - } - } -} - -/// Schemas to read records from: all under `--changed`, else those of unselected upstreams. -fn read_locations(facts: &BuildFacts, changed: bool) -> BTreeSet { - let by_key: HashMap<&NodeKey, &NodeFacts> = - facts.nodes.iter().map(|node| (&node.key, node)).collect(); - if changed { - return facts - .nodes - .iter() - .map(|node| Location::of(&node.relation)) - .collect(); - } - facts - .nodes - .iter() - .filter(|node| facts.runnable.contains(&node.key)) - .flat_map(|node| node.inputs.iter()) - .filter_map(|input| match input { - InputFact::Node(upstream) if !facts.runnable.contains(upstream) => by_key.get(upstream), - _ => None, - }) - .map(|node| Location::of(&node.relation)) - .collect() -} - -/// Everything the build needs from the recorder: the recorder itself and what `--changed` skipped. -pub(crate) struct Prepared<'py> { - pub(crate) recorder: Bound<'py, ConsumptionRecorder>, - pub(crate) decision: Option, - pub(crate) warnings: Vec, -} - -/// Read records, decide `--changed` skips, mint execution IDs and append `started` rows. -pub(crate) fn prepare<'py>( - (plan, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - (adapter, connection_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - (policy, full_refresh): (Option<&ChangedPolicy>, bool), - (run_id, base_on_node_complete): (String, Bound<'py, PyAny>), -) -> PyResult> { - let py: Python<'py> = plan.py(); - let facts: BuildFacts = build_facts((plan, project), (policy.is_some(), full_refresh))?; - let connection = ConsumptionConnection::new(adapter, connection_config); - let (history, warnings): (ConsumptionHistory, Vec) = - read_history(py, &connection, (&facts, policy)); - let decision: Option = - policy.map(|policy| changed_decision(&facts, &history, policy)); - let not_run: BTreeSet<&NodeKey> = decision.iter().flat_map(ChangedDecision::not_run).collect(); - let executions: HashMap = facts - .runnable - .iter() - .filter(|key| !not_run.contains(key)) - .map(|key| (key.clone(), Uuid::now_v7().to_string())) - .collect(); - let nodes: HashMap = facts - .nodes - .into_iter() - .map(|node| (node.key.clone(), node)) - .collect(); - let mut started: BTreeMap> = BTreeMap::new(); - for (key, execution_id) in &executions { - if let Some(node) = nodes.get(key) { - started - .entry(Location::of(&node.relation)) - .or_default() - .push(started_append(node, execution_id, &run_id)); - } - } - let mut failed_start: HashMap = HashMap::new(); - let mut tables: BTreeSet = BTreeSet::new(); - for (location, rows) in started { - let written: PyResult<()> = connection - .ensure_table(py, &location) - .and_then(|()| connection.append(py, &location, &rows)); - match written { - Ok(()) => { - tables.insert(location); - } - Err(error) => { - let message: String = error.to_string(); - for row in rows { - failed_start.insert(row.node, message.clone()); - } - } - } - } - connection.close(py)?; - let recorder = ConsumptionRecorder { - connection, - run_id, - nodes, - history, - executions, - failed_start, - base_on_node_complete: NodeCompleteHook::new(&base_on_node_complete), - recording: Mutex::new(Recording { - tables, - ..Recording::default() - }), - }; - Ok(Prepared { - recorder: Bound::new(py, recorder)?, - decision, - warnings, - }) -} - -/// What `plan --changed` previews: what a build would run now, read without recording anything. -pub(crate) fn preview_changed( - (plan, project): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - (adapter, connection_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - (policy, full_refresh): (&ChangedPolicy, bool), -) -> PyResult<(ChangedDecision, Vec)> { - let py: Python<'_> = plan.py(); - let facts: BuildFacts = build_facts((plan, project), (true, full_refresh))?; - let connection = ConsumptionConnection::new(adapter, connection_config); - let (history, warnings): (ConsumptionHistory, Vec) = - read_history(py, &connection, (&facts, Some(policy))); - connection.close(py)?; - Ok((changed_decision(&facts, &history, policy), warnings)) -} - -/// Recent executions the build reads, with older ones read back when deciding needs them. -fn read_history( - py: Python<'_>, - connection: &ConsumptionConnection, - (facts, policy): (&BuildFacts, Option<&ChangedPolicy>), -) -> (ConsumptionHistory, Vec) { - let mut warnings: Vec = Vec::new(); - let mut rows: Vec = Vec::new(); - for location in read_locations(facts, policy.is_some()) { - match connection.read_recent(py, &location) { - Ok(read) => rows.extend(read), - Err(error) => warnings.push(format!( - "could not read consumed inputs in schema {}: {error}; its models and seeds count as changed", - location.schema - )), - } - } - let history: ConsumptionHistory = consumption_history(&rows, RECENT_EXECUTIONS); - if policy.is_some_and(needs_read_back) { - let (history, read_back): (ConsumptionHistory, Vec) = - with_read_back(py, connection, history, &facts.nodes); - warnings.extend(read_back); - return (history, warnings); - } - (history, warnings) -} - -/// What `--changed` does with each runnable node. -fn changed_decision( - facts: &BuildFacts, - history: &ConsumptionHistory, - policy: &ChangedPolicy, -) -> ChangedDecision { - let mut reasons: Vec<(NodeKey, RunReason)> = Vec::new(); - let mut skipped: Vec = Vec::new(); - let mut held_back: Vec<(NodeKey, HeldBack)> = Vec::new(); - for (key, outcome) in changed_run_set(&facts.nodes, history, policy) { - if !facts.runnable.contains(&key) { - continue; - } - match outcome { - ChangedOutcome::Runs(reason) => reasons.push((key, reason)), - ChangedOutcome::Unchanged => skipped.push(key), - ChangedOutcome::HeldBack(why) => held_back.push((key, why)), - } - } - ChangedDecision { - policy: policy.clone(), - run: reasons.len(), - skipped, - held_back, - reasons, - } -} - -/// Whether deciding needs older executions: to tell definition from data, or to attribute a change. -fn needs_read_back(policy: &ChangedPolicy) -> bool { - policy.counts.len() == 1 || matches!(policy.reach, Reach::Job(_)) -} - -/// `history` plus every older upstream execution its builds read, followed back until none is left. -fn with_read_back( - py: Python<'_>, - connection: &ConsumptionConnection, - history: ConsumptionHistory, - nodes: &[NodeFacts], -) -> (ConsumptionHistory, Vec) { - let mut warnings: BTreeSet = BTreeSet::new(); - let history: ConsumptionHistory = read_back_history(history, nodes, |references| { - let (rows, failures) = read_references(py, connection, references); - warnings.extend(failures); - rows - }); - (history, warnings.into_iter().collect()) -} - -/// The rows of these executions; a schema that cannot be read leaves its rebuilds counted. -fn read_references( - py: Python<'_>, - connection: &ConsumptionConnection, - references: &[(RelationKey, String)], -) -> (Vec, Vec) { - let mut by_location: BTreeMap> = BTreeMap::new(); - for (relation, id) in references { - by_location - .entry(Location::of(relation)) - .or_default() - .push(id.as_str()); - } - let mut rows: Vec = Vec::new(); - let mut warnings: Vec = Vec::new(); - for (location, ids) in by_location { - match connection.read_executions(py, &location, &ids) { - Ok(read) => rows.extend(read), - Err(error) => warnings.push(format!( - "could not read earlier executions in schema {}: {error}; rebuilt upstreams there count as changed", - location.schema - )), - } - } - (rows, warnings) -} - -/// Why `--changed` holds a node back, as one line for verbose output. -pub(crate) fn held_back_text(why: &HeldBack) -> String { - match why { - HeldBack::RefreshedBy(jobs) => format!( - "job {} refreshes it, and the change is not in that job", - jobs.join(", ") - ), - HeldBack::ReadsHeldBack(key) => format!("reads held-back {} {}", key.kind, key.name), - } -} - -/// Why `--changed` runs a node, as one line for verbose output. -pub(crate) fn run_reason_text(reason: &RunReason) -> String { - match reason { - RunReason::Forced(ForcedRun::PlannerChange(kind)) => format!("planner change ({kind})"), - RunReason::Forced(ForcedRun::FullRefresh) => "full refresh".to_owned(), - RunReason::Forced(ForcedRun::RelationMissing) => "relation missing".to_owned(), - RunReason::Forced(ForcedRun::CursorInterval) => "cursor interval requested".to_owned(), - RunReason::Forced(ForcedRun::InputsNotEnumerable) => { - "inputs cannot be enumerated".to_owned() - } - RunReason::NoRecord => "no completed record".to_owned(), - RunReason::Unfinished => "last execution did not finish".to_owned(), - RunReason::AmbiguousOrder => "record order is ambiguous".to_owned(), - RunReason::DefinitionChanged => "definition changed".to_owned(), - RunReason::InputsChanged => "inputs changed".to_owned(), - RunReason::InputChanged(input) => format!("input {input} changed"), - RunReason::UpstreamDefinitionChanged(input) => { - format!("upstream {input} was rebuilt with a definition change") - } - RunReason::InputUnknown(input) => format!("input {input} has no known version"), - RunReason::UpstreamRuns(key) => format!("upstream {} {} runs", key.kind, key.name), - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index b19e980701..6b3a3776b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -15,7 +15,6 @@ use crate::bindings::_helpers::compile_driver::discovery::discover_project_input use crate::bindings::_helpers::cost_report::functions::read_cost_run; use crate::bindings::_helpers::cost_report::functions::resolve_cost_run; use crate::bindings::_helpers::cost_report::functions::write_cost_run; -use crate::bindings::_helpers::cost_telemetry::values::cost_aware; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -26,6 +25,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_front::compile_attachment::main::build_effective_target_config::build_effective_target_config; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::cost_context::main::cost_aware::cost_aware; /// Collection budget of an explicit `sqb cost` refresh, which a user waits for on purpose. const REFRESH_BUDGET_SECONDS: f64 = 300.0; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs index 8935152ffc..39ea17272d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs @@ -16,12 +16,12 @@ use crate::bindings::_helpers::cost_report::attribution::report_cost_status; use crate::bindings::_helpers::cost_report::publication::publish_resource_costs; use crate::bindings::_helpers::cost_report::records::cost_run; use crate::bindings::_helpers::cost_report::store; -use crate::bindings::_helpers::cost_telemetry::values::cost_aware; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cost_context::main::cost_aware::cost_aware; const LOGGER_NAME: &str = "sqlbuild.cli.commands._helpers.cost.collection"; const COLLECTION_LOGGER_NAME: &str = "sqlbuild.cost"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs index f3cd30f4c2..ff7d020b11 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs @@ -10,15 +10,15 @@ use sqlbuild_core::cost_allocation::models::{ }; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::cost_telemetry::values::{ - instant_micros, native_decimal, python_decimal, -}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cost_context::main::instant_micros::instant_micros; +use sqlbuild_python_front::cost_context::main::native_decimal::native_decimal; +use sqlbuild_python_front::cost_context::main::python_decimal::python_decimal; const PENDING_MESSAGE: &str = "Snowflake query history is not complete yet."; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs index 55ccc7814e..9539893b06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs @@ -8,7 +8,7 @@ use pyo3::sync::PyOnceLock; use pyo3::types::PyTypeMethods; use pyo3::{pyclass, pymethods}; -use crate::bindings::_helpers::cost_telemetry::values::process_object; +use sqlbuild_python_front::cost_context::main::process_object::process_object; const HEALTH_ATTRIBUTE: &str = "COST_TELEMETRY_HEALTH"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs index 0f8c8fc14c..b02639c427 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs @@ -16,7 +16,6 @@ use sqlbuild_core::json::main::dumps::dumps; use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; use crate::bindings::_helpers::cost_telemetry::health::mark_ledger_failure; -use crate::bindings::_helpers::cost_telemetry::values::process_object; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -26,6 +25,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::cost_context::main::process_object::process_object; use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; static LEDGER_LOCK: PyOnceLock> = PyOnceLock::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs index 89f46b212d..de3d311170 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs @@ -1,7 +1,5 @@ pub(crate) mod allocation; -pub(crate) mod context; pub(crate) mod health; pub(crate) mod ledger; pub(crate) mod registration; pub(crate) mod snowflake; -pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs index 4ce5de8966..f9332a6bf3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs @@ -2,11 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::cost_telemetry::{allocation, context, health, ledger, snowflake}; +use crate::bindings::_helpers::cost_telemetry::{allocation, health, ledger, snowflake}; +use sqlbuild_python_front::cost_context::main::register_cost_context::register_cost_context; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { allocation::register(module)?; - context::register(module)?; + register_cost_context(module)?; health::register(module)?; ledger::register(module)?; snowflake::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs deleted file mode 100644 index 5a415eec9a..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs +++ /dev/null @@ -1,90 +0,0 @@ -//! Python `Decimal` and `datetime` values as exact native decimals and microsecond instants. - -use pyo3::IntoPyObjectExt; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyTuple}; -use sqlbuild_core::cost_allocation::models::PyDecimal; - -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; -use sqlbuild_python_front::state_rows::main::event_rows::offset_aware::offset_aware; - -const DECIMAL_MODULE: &str = "decimal"; -/// The methods an adapter with warehouse cost telemetry implements. -const COST_AWARE_METHODS: [&str; 2] = ["cost_capability", "collect_run_cost"]; - -/// Whether `adapter` implements the cost telemetry methods. -pub(crate) fn cost_aware(adapter: &Bound<'_, PyAny>) -> PyResult { - for name in COST_AWARE_METHODS { - if !adapter.hasattr(name)? || native_field(adapter, name)?.is_none() { - return Ok(false); - } - } - Ok(true) -} - -/// A finite non-negative `Decimal` as its exact digits and exponent. -pub(crate) fn native_decimal(value: &Bound<'_, PyAny>) -> PyResult { - let (sign, digits, exponent): (u8, Vec, Bound<'_, PyAny>) = - value.call_method0("as_tuple")?.extract()?; - let Ok(exponent) = exponent.extract::() else { - return Err(PyValueError::new_err("cost decimals must be finite")); - }; - if sign != 0 && digits.iter().any(|digit| *digit != 0) { - return Err(PyValueError::new_err("cost decimals must not be negative")); - } - let leading: usize = digits.iter().take_while(|digit| **digit == 0).count(); - let digits: Vec = if leading == digits.len() { - vec![0] - } else { - digits[leading..].to_vec() - }; - Ok(PyDecimal { digits, exponent }) -} - -/// `Decimal((0, digits, exponent))`. -pub(crate) fn python_decimal<'py>( - py: Python<'py>, - value: &PyDecimal, -) -> PyResult> { - let parts: Bound<'py, PyTuple> = PyTuple::new( - py, - [ - 0_i64.into_bound_py_any(py)?, - PyTuple::new(py, &value.digits)?.into_any(), - value.exponent.into_bound_py_any(py)?, - ], - )?; - imported(py, DECIMAL_MODULE, "Decimal")?.call1((parts,)) -} - -/// Microseconds since the epoch: UTC for aware values, the wall clock for naive ones. -pub(crate) fn instant_micros(value: &Bound<'_, PyAny>) -> PyResult { - let py: Python<'_> = value.py(); - let datetime: Bound<'_, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; - let epoch: Bound<'_, PyAny> = if offset_aware(value)? { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("tzinfo", imported(py, DATETIME_MODULE, "UTC")?)?; - datetime.call((1970, 1, 1), Some(&options))? - } else { - datetime.call1((1970, 1, 1))? - }; - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("microseconds", 1)?; - let micro: Bound<'_, PyAny> = - imported(py, DATETIME_MODULE, "timedelta")?.call((), Some(&options))?; - value.sub(epoch)?.floor_div(micro)?.extract() -} - -/// The process-wide Python object in `cell`, built by `build` on first use. -pub(crate) fn process_object<'py>( - cell: &'static PyOnceLock>, - py: Python<'py>, - build: impl FnOnce() -> PyResult>, -) -> PyResult<&'py Bound<'py, PyAny>> { - cell.get_or_try_init(py, || Ok::, PyErr>(build()?.unbind())) - .map(|object| object.bind(py)) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs index 509bfa4aca..b0968bf304 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs @@ -8,7 +8,6 @@ use sqlbuild_core::cli_text::main::format_dbt_interop_plan::format_dbt_interop_p use sqlbuild_core::cli_text::main::format_dbt_interop_plan_json::format_dbt_interop_plan_json; use sqlbuild_core::cli_text::models::{DbtInteropPlanText, DbtLsNodeText, DisplayLimit}; -use crate::bindings::_helpers::plan_output::plan_text::format_plan; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; @@ -18,6 +17,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_planner::plan_output::main::format_plan::format_plan; const BUILD_OPTIONS: (&[&str], &[&str]) = ( &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 3ba6d26f14..69c050065a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -11,8 +11,6 @@ use crate::bindings::_helpers::command_runtime::connection::{ use crate::bindings::_helpers::command_runtime::warehouses::apply_command_warehouse; use crate::bindings::_helpers::compile_driver::deferral::resolve_model_deferral_inputs; use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; -use crate::bindings::_helpers::planner_entries::display_plan::build_display_only_sqlbuild_plan; -use crate::bindings::_helpers::planner_entries::execution_plan::build_execution_plan; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -28,6 +26,8 @@ use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; +use sqlbuild_python_planner::planner_entries::main::plans::build_display_only_sqlbuild_plan::build_display_only_sqlbuild_plan; +use sqlbuild_python_planner::planner_entries::main::plans::build_execution_plan::build_execution_plan; const BUILD_OPTIONS: (&[&str], &[&str]) = (&["environment", "request", "hooks"], &[]); const ATTACH_OPTIONS: (&[&str], &[&str]) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 91312c9289..c58a5b69ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -17,9 +17,6 @@ use sqlbuild_orchestration::runs::models::NodeOutcome; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::freshness::freshness_observe::{PreviousState, observe_freshness}; -use crate::bindings::_helpers::freshness::observation::Observer; -use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; @@ -27,6 +24,11 @@ use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_planner::freshness::main::observe_freshness::observe_freshness; +use sqlbuild_python_planner::freshness::main::read_latest::read_latest; +use sqlbuild_python_planner::freshness::models::Observer; +use sqlbuild_python_planner::freshness::models::PreviousState; +use sqlbuild_python_planner::freshness::models::StateRead; use sqlbuild_python_project::planner_selection::main::expand_project_upstream_keys::expand_project_upstream_keys; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs index 98a0933a17..e5f2b2a5fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs @@ -1,9 +1,2 @@ -pub(crate) mod exception_types; pub(crate) mod freshness_command; -pub(crate) mod freshness_observe; -pub(crate) mod observation; -pub(crate) mod planning; -pub(crate) mod publication; pub(crate) mod registration; -pub(crate) mod state_io; -pub(crate) mod support; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs index d81d40f9e0..e69c8470a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs @@ -2,15 +2,18 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::freshness::{ - exception_types, freshness_observe, observation, planning, publication, state_io, -}; +use sqlbuild_python_planner::freshness::main::register_exception_types::register_exception_types; +use sqlbuild_python_planner::freshness::main::register_freshness_observe::register_freshness_observe; +use sqlbuild_python_planner::freshness::main::register_observation::register_observation; +use sqlbuild_python_planner::freshness::main::register_planning::register_planning; +use sqlbuild_python_planner::freshness::main::register_publication::register_publication; +use sqlbuild_python_planner::freshness::main::register_state_io::register_state_io; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - exception_types::register(module)?; - freshness_observe::register(module)?; - observation::register(module)?; - planning::register(module)?; - publication::register(module)?; - state_io::register(module) + register_exception_types(module)?; + register_freshness_observe(module)?; + register_observation(module)?; + register_planning(module)?; + register_publication(module)?; + register_state_io(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs index b36f00df8e..04c11cd605 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs @@ -25,8 +25,8 @@ use crate::bindings::_helpers::janitor::janitor_keys::{ Folded, Key, SchemaKey, archive_timestamp, days, folded_schema, optional, record, relation_age_timestamp, sorted_schemas, text, timestamp_fields, tuple, utc_datetime, }; -use crate::bindings::_helpers::planner_scenario::artifacts::is_scenario_artifact_physical_name; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_planner::planner_scenario::main::is_scenario_artifact_physical_name::is_scenario_artifact_physical_name; use sqlbuild_python_project::relation_naming::main::shared_error::shared_error; /// Protection rules for one plan. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs deleted file mode 100644 index 411f03bc91..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs +++ /dev/null @@ -1,87 +0,0 @@ -//! The native lifecycle event catalogue: known types, terminality and catalogued values. - -use pyo3::exceptions::PyKeyError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyFrozenSet, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::lifecycle_events::constants::{ - CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, HOOK_PHASES, - HOOK_TYPES, LIFECYCLE_EVENT_SCHEMA_VERSIONS, OPERATION_ADAPTERS, OPERATION_KINDS, - OPERATION_METADATA_FIELDS, OPERATION_NAMES, OPERATION_PHASES, OPERATION_SCOPES, - OPERATION_STRATEGIES, OPERATION_TARGET_KINDS, RESOURCE_SKIP_CODES, RESOURCE_SKIP_MODES, -}; -use sqlbuild_core::lifecycle_events::main::catalog::catalog; -use sqlbuild_core::lifecycle_events::models::EventDefinition; - -/// The definition of `event_type` in the `schema_version` catalogue, if both are catalogued. -pub(crate) fn definition( - schema_version: &Bound<'_, PyAny>, - event_type: &Bound<'_, PyAny>, -) -> PyResult> { - let (Ok(version), Ok(name)) = ( - schema_version.extract::(), - event_type.downcast::(), - ) else { - return Ok(None); - }; - let name: String = name.extract()?; - Ok(catalog(version) - .and_then(|entries| entries.into_iter().find(|entry| entry.event_type == name))) -} - -/// Whether `event_type` is catalogued for `schema_version`. -#[pyfunction] -#[pyo3(signature = (*, schema_version, event_type))] -fn lifecycle_event_catalogued( - schema_version: &Bound<'_, PyAny>, - event_type: &Bound<'_, PyAny>, -) -> PyResult { - Ok(definition(schema_version, event_type)?.is_some()) -} - -/// Whether a catalogued event closes its correlated lifecycle scope; `KeyError` otherwise. -#[pyfunction] -#[pyo3(signature = (*, schema_version, event_type))] -fn lifecycle_event_terminal( - schema_version: &Bound<'_, PyAny>, - event_type: &Bound<'_, PyAny>, -) -> PyResult { - definition(schema_version, event_type)? - .map(|entry| entry.terminal) - .ok_or_else(|| PyKeyError::new_err(event_type.clone().unbind())) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - let py = module.py(); - module.add( - "CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION", - CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, - )?; - module.add( - "CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION", - CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, - )?; - module.add( - "LIFECYCLE_EVENT_SCHEMA_VERSIONS", - PyFrozenSet::new(py, LIFECYCLE_EVENT_SCHEMA_VERSIONS)?, - )?; - let sets: [(&str, &[&str]); 12] = [ - ("HOOK_PHASES", &HOOK_PHASES), - ("HOOK_TYPES", &HOOK_TYPES), - ("OPERATION_ADAPTERS", &OPERATION_ADAPTERS), - ("OPERATION_KINDS", &OPERATION_KINDS), - ("OPERATION_METADATA_FIELDS", &OPERATION_METADATA_FIELDS), - ("OPERATION_NAMES", &OPERATION_NAMES), - ("OPERATION_PHASES", &OPERATION_PHASES), - ("OPERATION_SCOPES", &OPERATION_SCOPES), - ("OPERATION_STRATEGIES", &OPERATION_STRATEGIES), - ("OPERATION_TARGET_KINDS", &OPERATION_TARGET_KINDS), - ("RESOURCE_SKIP_CODES", &RESOURCE_SKIP_CODES), - ("RESOURCE_SKIP_MODES", &RESOURCE_SKIP_MODES), - ]; - for (name, values) in sets { - module.add(name, PyFrozenSet::new(py, values)?)?; - } - module.add_function(wrap_pyfunction!(lifecycle_event_catalogued, module)?)?; - module.add_function(wrap_pyfunction!(lifecycle_event_terminal, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs index ccd5f51ee1..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs @@ -1,5 +1 @@ -pub(crate) mod catalog; -pub(crate) mod envelopes; -pub(crate) mod payloads; pub(crate) mod registration; -pub(crate) mod validation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs index 1bb8d62067..d1cb5c0a9f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs @@ -2,10 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::lifecycle_events::{catalog, envelopes, validation}; +use sqlbuild_python_planner::lifecycle_events::main::register_catalog::register_catalog; +use sqlbuild_python_planner::lifecycle_events::main::register_envelopes::register_envelopes; +use sqlbuild_python_planner::lifecycle_events::main::register_validation::register_validation; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - catalog::register(module)?; - envelopes::register(module)?; - validation::register(module) + register_catalog(module)?; + register_envelopes(module)?; + register_validation(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 07b778128a..6b5caa16b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -31,7 +31,6 @@ use crate::bindings::_helpers::load::load_progress::{ use crate::bindings::_helpers::load::load_selection::{ LoadProject, LoaderDefaults, loader_defaults, loader_write_schemas, target_vars, }; -use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::result_output::documents::format_load_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; @@ -55,6 +54,7 @@ use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_use use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; use sqlbuild_python_front::discovery::models::SecretPolicy; +use sqlbuild_python_planner::planner_resolution::main::cursor::type_cursor_overrides::type_cursor_overrides; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 879f3acf19..2f165dfbf8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -16,7 +16,6 @@ use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; -use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::orchestration_reads::main::environment_hash::environment_hash; use sqlbuild_python_base::records::main::fields::field::field; @@ -24,6 +23,7 @@ use sqlbuild_python_front::discovery::main::project::named_node_rows::named_node use sqlbuild_python_front::discovery::main::project::project_facts::project_facts; use sqlbuild_python_front::discovery::main::project::validation_error::validation_error; use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; +use sqlbuild_python_planner::planner_identity::main::direct_model_version_identities::direct_model_version_identities; use sqlbuild_python_project::compiled_views::models::NativeProjectView; use sqlbuild_python_project::graph::models::ProjectSelection; use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/mod.rs index e1b436ef3d..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/mod.rs @@ -1,4 +1 @@ -pub(crate) mod conversion; -pub(crate) mod projection; pub(crate) mod registration; -pub(crate) mod statements; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/registration.rs index b114a939df..73e1e8c28a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::microbatches::{projection, statements}; +use sqlbuild_python_planner::microbatches::main::register_projection::register_projection; +use sqlbuild_python_planner::microbatches::main::register_statements::register_statements; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - projection::register(module)?; - statements::register(module) + register_projection(module)?; + register_statements(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 9ad0e502dd..e02a4eae34 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -20,7 +20,6 @@ pub(crate) mod compile_outputs; pub(crate) mod compiled_project; pub(crate) mod compiled_views; pub(crate) mod compute_logs; -pub(crate) mod consumption; pub(crate) mod contract_command; pub(crate) mod contracts; pub(crate) mod cost_command; @@ -57,7 +56,6 @@ pub(crate) mod manifest; pub(crate) mod microbatches; pub(crate) mod model_config; pub(crate) mod native_host; -pub(crate) mod observations; pub(crate) mod output_capture; pub(crate) mod plan_command; pub(crate) mod plan_output; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/statement_report.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/statement_report.rs index f34809d85d..eeb9cc5d32 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/statement_report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/statement_report.rs @@ -14,7 +14,7 @@ use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_python_base::build_progress::main::progress::current_projector::current_projector; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use crate::bindings::_helpers::cost_telemetry::values::cost_aware; +use sqlbuild_python_front::cost_context::main::cost_aware::cost_aware; /// The JSON document section the report adds. const STATEMENTS_SECTION: &str = "statements"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 61e4e26117..29d7f8f6dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -34,8 +34,6 @@ use sqlbuild_state::state_tables::main::read_history_sql::read_history_sql; use sqlbuild_state::state_tables::models::HistoryRead; use crate::bindings::_helpers::command_runtime::connection::project_connection_config; -use crate::bindings::_helpers::freshness::state_io::row_record; -use crate::bindings::_helpers::freshness::support::presumed_utc_datetime; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; @@ -49,6 +47,8 @@ use sqlbuild_python_base::warehouse_access::main::read_catalog::read_catalog; use sqlbuild_python_base::warehouse_access::main::read_statistics::read_statistics; use sqlbuild_python_front::authored_values::main::sql_value::sql_value; use sqlbuild_python_front::authored_values::models::SqlValueClasses; +use sqlbuild_python_planner::freshness::main::presumed_utc_datetime::presumed_utc_datetime; +use sqlbuild_python_planner::freshness::main::row_record::row_record; use sqlbuild_python_project::compiled_project::main::plain_value::plain_value; use sqlbuild_python_project::fingerprints::main::row_fingerprint::row_fingerprint; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs deleted file mode 100644 index b33192b5fd..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/observations/source_capture.rs +++ /dev/null @@ -1,47 +0,0 @@ -//! The expression-source columns a build's planning described, kept for its observations. - -use std::cell::RefCell; - -use serde_json::{Map, Value}; -use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; -use sqlbuild_state::observations::constants::{ - SOURCE_COLUMNS_NAMES_KEY, SOURCE_COLUMNS_SQL_HASH_KEY, -}; - -/// Each described source's name and observation value, in the order planning described them. -pub(crate) type DescribedSources = Vec<(String, String)>; - -thread_local! { - /// Per thread, since a build plans and records on the thread that runs it; `None` when not armed. - static CAPTURED: RefCell> = const { RefCell::new(None) }; -} - -/// Start capturing described source columns; only a build arms it, so `sqb plan` records nothing. -pub(crate) fn arm_source_capture() { - CAPTURED.with(|slot| *slot.borrow_mut() = Some(Vec::new())); -} - -/// Keep one source's described SQL hash and column names when a build armed the capture. -pub(crate) fn capture_source_columns(source: &str, sql: &str, columns: &[String]) { - CAPTURED.with(|slot| { - if let Some(captured) = slot.borrow_mut().as_mut() { - let mut value: Map = Map::new(); - value.insert( - SOURCE_COLUMNS_SQL_HASH_KEY.to_owned(), - Value::String(sha256_hex(sql.as_bytes())), - ); - value.insert( - SOURCE_COLUMNS_NAMES_KEY.to_owned(), - Value::from(columns.to_vec()), - ); - captured.push((source.to_owned(), Value::Object(value).to_string())); - } - }); -} - -/// The captured source columns, disarming the capture. -pub(crate) fn take_source_capture() -> DescribedSources { - CAPTURED - .with(|slot| slot.borrow_mut().take()) - .unwrap_or_default() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/changed.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/changed.rs index aafe27a81f..27c939e00c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/changed.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/changed.rs @@ -6,8 +6,10 @@ use sqlbuild_core::cli_text::main::format_tree_connector::format_tree_connector; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_state::consumption::models::{HeldBack, NodeKey, RunReason}; -use crate::bindings::_helpers::consumption::policy::{kind_value, reach_value}; -use crate::bindings::_helpers::consumption::recorder::{ChangedDecision, run_reason_text}; +use sqlbuild_python_planner::consumption::main::kind_value::kind_value; +use sqlbuild_python_planner::consumption::main::reach_value::reach_value; +use sqlbuild_python_planner::consumption::main::run_reason_text::run_reason_text; +use sqlbuild_python_planner::consumption::models::ChangedDecision; fn node(key: &NodeKey) -> String { format!("{}:{}", key.kind, key.name) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 2ffe94163e..b5b8167d49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -20,12 +20,8 @@ use crate::bindings::_helpers::command_runtime::preview_target::validate_preview use crate::bindings::_helpers::command_runtime::runtime_errors::raised; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; -use crate::bindings::_helpers::consumption::policy::{changed_policy, policy_line}; -use crate::bindings::_helpers::consumption::recorder::{ChangedDecision, preview_changed}; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::plan_command::changed::{changed_json, changed_text}; -use crate::bindings::_helpers::plan_output::plan_json::{CHANGED_KEY, format_plan_json}; -use crate::bindings::_helpers::plan_output::plan_text::format_plan; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -37,6 +33,13 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_front::compile_attachment::main::build_effective_settings_config::build_effective_settings_config; +use sqlbuild_python_planner::consumption::main::changed_policy::changed_policy; +use sqlbuild_python_planner::consumption::main::policy_line::policy_line; +use sqlbuild_python_planner::consumption::main::preview_changed::preview_changed; +use sqlbuild_python_planner::consumption::models::ChangedDecision; +use sqlbuild_python_planner::plan_output::constants::CHANGED_KEY; +use sqlbuild_python_planner::plan_output::main::format_plan::format_plan; +use sqlbuild_python_planner::plan_output::main::format_plan_json::format_plan_json; const COMMAND_NAME: &str = "plan"; const DEFAULT_SECTION_ENTRIES: i64 = 50; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs deleted file mode 100644 index 4cd6d088fe..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs +++ /dev/null @@ -1,257 +0,0 @@ -//! Plan and progress display fragments over planner entries and cursor values. - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::plan_display::constants::{ - DEFAULT_CURSOR_GRAIN, INTEGER_CURSOR, TIMESTAMP_CURSOR, -}; -use sqlbuild_core::plan_display::main::cursor_bound_display::cursor_bound_display as bound_display; -use sqlbuild_core::plan_display::main::effective_microbatch_batch_size::effective_microbatch_batch_size; -use sqlbuild_core::plan_display::main::format_no_input_rows::format_no_input_rows as no_input_rows; -use sqlbuild_core::plan_display::main::format_waiting_on_empty_inputs::format_waiting_on_empty_inputs as waiting_on_empty_inputs; -use sqlbuild_core::plan_display::main::model_materialization_label::model_materialization_label as materialization_label; -use sqlbuild_core::plan_display::models::{ModelLabelFacts, ParsedBound}; - -use crate::bindings::_helpers::plan_output::values::{optional_text, text}; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; -use sqlbuild_python_front::cursor::main::scalars::to_exclusive::to_exclusive; -use sqlbuild_python_front::cursor::main::scalars::to_inclusive::to_inclusive; -use sqlbuild_python_front::cursor::main::scalars::try_parse::try_parse; - -const DATETIME_MODULE: &str = "datetime"; - -fn label_facts(entry: &Bound<'_, PyAny>) -> PyResult { - let field = - |name: &str| -> PyResult> { optional_text(&native_field(entry, name)?) }; - Ok(ModelLabelFacts { - materialization_type: text(&native_field(entry, "materialization_type")?)?, - action: text(&native_field(entry, "action")?)?, - custom_materialization_name: field("custom_materialization_name")?, - incremental_strategy: field("incremental_strategy")?, - incremental_mode: field("incremental_mode")?, - cursor_type: field("cursor_type")?, - snapshot_strategy: field("snapshot_strategy")?, - historical: !native_field(entry, "observed_at_column")?.is_none(), - historical_input: field("historical_input")?, - }) -} - -pub(crate) fn optional_facts(entry: &Bound<'_, PyAny>) -> PyResult> { - if entry.is_none() { - return Ok(None); - } - label_facts(entry).map(Some) -} - -/// The cursor type, defaulting to timestamp, and its grain object (`None` for integers). -fn cursor_typing<'py>( - cursor_type: &Bound<'py, PyAny>, - cursor_grain: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - let py: Python<'py> = cursor_type.py(); - let effective: Bound<'py, PyAny> = if cursor_type.is_truthy()? { - cursor_type.clone() - } else { - PyString::new(py, TIMESTAMP_CURSOR).into_any() - }; - if effective.eq(INTEGER_CURSOR)? { - return Ok((effective, py.None().into_bound(py))); - } - let grain: Bound<'py, PyAny> = if cursor_grain.is_truthy()? { - cursor_grain.clone() - } else { - PyString::new(py, DEFAULT_CURSOR_GRAIN).into_any() - }; - let grain: Bound<'py, PyAny> = enum_from_value(py, RecordId::CursorGrain, grain)?; - Ok((effective, grain)) -} - -/// The last cursor value included by an exclusive end bound. -#[pyfunction] -#[pyo3(signature = (*, end, cursor_type, cursor_grain))] -pub(crate) fn inclusive_cursor_end( - end: &Bound<'_, PyAny>, - cursor_type: &Bound<'_, PyAny>, - cursor_grain: &Bound<'_, PyAny>, -) -> PyResult { - let (effective, grain) = cursor_typing(cursor_type, cursor_grain)?; - let Some(parsed) = try_parse(end, &effective)? else { - return end.extract(); - }; - render_value(&to_inclusive(&parsed, &grain)?) -} - -/// A fully bounded inclusive operator override as an executable interval. -pub(crate) fn resolve_bounded_cursor_override<'py>( - start_cursor_override: &Bound<'py, PyAny>, - end_cursor_override: &Bound<'py, PyAny>, - cursor_type: &Bound<'py, PyAny>, - cursor_grain: &Bound<'py, PyAny>, -) -> PyResult>> { - let py: Python<'py> = start_cursor_override.py(); - if start_cursor_override.is_none() || end_cursor_override.is_none() { - return Ok(None); - } - let (effective, grain) = cursor_typing(cursor_type, cursor_grain)?; - let (Some(start), Some(end)) = ( - try_parse(start_cursor_override, &effective)?, - try_parse(end_cursor_override, &effective)?, - ) else { - return Ok(None); - }; - new_record_with( - py, - RecordId::CursorBounds, - &[("start", start), ("end", to_exclusive(&end, &grain)?)], - ) - .map(Some) -} - -fn parsed_bound(value: &Bound<'_, PyAny>) -> PyResult> { - let py: Python<'_> = value.py(); - let parsed: Bound<'_, PyAny> = - match imported(py, DATETIME_MODULE, "datetime")?.call_method1("fromisoformat", (value,)) { - Ok(parsed) => parsed, - Err(error) if error.is_instance_of::(py) => return Ok(None), - Err(error) => return Err(error), - }; - let mut midnight: bool = true; - for part in ["hour", "minute", "second", "microsecond"] { - midnight = midnight && native_field(&parsed, part)?.extract::()? == 0; - } - Ok(Some(ParsedBound { - date: parsed - .call_method0("date")? - .call_method0("isoformat")? - .extract()?, - midnight, - })) -} - -/// A cursor bound for display, collapsing a whole-day midnight timestamp to a date. -#[pyfunction] -#[pyo3(signature = (*, value, cursor_type, cursor_grain))] -pub(crate) fn cursor_bound_display( - value: &Bound<'_, PyAny>, - cursor_type: &Bound<'_, PyAny>, - cursor_grain: &Bound<'_, PyAny>, -) -> PyResult { - let cursor_type: Option = optional_text(cursor_type)?; - let raw: String = value.extract()?; - let parsed: Option = if cursor_type.as_deref() == Some(INTEGER_CURSOR) { - None - } else { - parsed_bound(value)? - }; - Ok(bound_display( - &raw, - ( - cursor_type.as_deref(), - optional_text(cursor_grain)?.as_deref(), - ), - parsed.as_ref(), - )) -} - -/// The full materialization label used in plan summaries. -pub(crate) fn model_materialization_label(entry: &Bound<'_, PyAny>) -> PyResult { - Ok(materialization_label(&label_facts(entry)?)) -} - -fn names(input_names: &Bound<'_, PyAny>) -> PyResult> { - input_names - .try_iter()? - .map(|name| name?.extract()) - .collect() -} - -/// Explain that a cursor window is empty because its inputs have no rows. -pub(crate) fn format_no_input_rows(input_names: &Bound<'_, PyAny>) -> PyResult { - Ok(no_input_rows(&names(input_names)?)) -} - -/// Explain that a cursor window waits on empty inputs while other inputs have rows. -#[pyfunction] -pub(crate) fn format_waiting_on_empty_inputs(input_names: &Bound<'_, PyAny>) -> PyResult { - Ok(waiting_on_empty_inputs(&names(input_names)?)) -} - -/// The explicit effective batch-size token resolved against the replay grain. -#[pyfunction] -#[pyo3(signature = (*, batch_size, effective_grain))] -pub(crate) fn resolve_effective_microbatch_batch_size( - py: Python<'_>, - batch_size: &str, - effective_grain: &str, -) -> Result { - effective_microbatch_batch_size(batch_size, effective_grain).map_err(|error| { - record_class(py, RecordId::CursorGrain) - .and_then(|grain| grain.call1((error.0,))) - .err() - .unwrap_or_else(|| PyValueError::new_err("unknown cursor grain")) - }) -} - -fn as_datetime<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = value.py(); - let inner: Bound<'py, PyAny> = native_field(value, "value")?; - if value.is_instance(&record_class(py, RecordId::TimestampValue)?)? { - return Ok(inner); - } - let datetime: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; - let midnight: Bound<'py, PyAny> = native_field(&datetime, "min")?.call_method0("time")?; - datetime.call_method1("combine", (inner, midnight)) -} - -fn presume_utc<'py>(value: Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = value.py(); - if !native_field(&value, "tzinfo")?.is_none() { - return Ok(value); - } - let options: Bound<'py, PyDict> = PyDict::new(py); - pyo3::types::PyDictMethods::set_item( - &options, - "tzinfo", - imported(py, DATETIME_MODULE, "UTC")?, - )?; - value.call_method("replace", (), Some(&options)) -} - -/// Comparable bounds, presuming a naive side UTC when the other is aware. -#[pyfunction] -#[pyo3(signature = (*, start, end))] -pub(crate) fn aligned_timestamp_bounds<'py>( - start: &Bound<'py, PyAny>, - end: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = start.py(); - let mut start_at: Bound<'py, PyAny> = as_datetime(start)?; - let mut end_at: Bound<'py, PyAny> = as_datetime(end)?; - if native_field(&start_at, "tzinfo")?.is_none() != native_field(&end_at, "tzinfo")?.is_none() { - start_at = presume_utc(start_at)?; - end_at = presume_utc(end_at)?; - } - PyTuple::new(py, [start_at, end_at]) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(inclusive_cursor_end, module)?)?; - module.add_function(wrap_pyfunction!(cursor_bound_display, module)?)?; - module.add_function(wrap_pyfunction!(format_waiting_on_empty_inputs, module)?)?; - module.add_function(wrap_pyfunction!( - resolve_effective_microbatch_batch_size, - module - )?)?; - module.add_function(wrap_pyfunction!(aligned_timestamp_bounds, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs index a4063abaf5..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs @@ -1,9 +1 @@ -pub(crate) mod cursor_details; -pub(crate) mod display; -pub(crate) mod migrations; -pub(crate) mod plan_json; -pub(crate) mod plan_text; -pub(crate) mod python_identity; pub(crate) mod registration; -pub(crate) mod safety; -pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs index 96c0593e33..285db800b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs @@ -2,12 +2,16 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::plan_output::{display, migrations, plan_json, plan_text, safety}; +use sqlbuild_python_planner::plan_output::main::register_display::register_display; +use sqlbuild_python_planner::plan_output::main::register_migrations::register_migrations; +use sqlbuild_python_planner::plan_output::main::register_plan_json::register_plan_json; +use sqlbuild_python_planner::plan_output::main::register_plan_text::register_plan_text; +use sqlbuild_python_planner::plan_output::main::register_safety::register_safety; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - display::register(module)?; - migrations::register(module)?; - plan_json::register(module)?; - plan_text::register(module)?; - safety::register(module) + register_display(module)?; + register_migrations(module)?; + register_plan_json(module)?; + register_plan_text(module)?; + register_safety(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs deleted file mode 100644 index 62a673e840..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs +++ /dev/null @@ -1,167 +0,0 @@ -//! Structured future-cursor and maximum-start safety evidence for plan and result JSON. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; - -use crate::bindings::_helpers::plan_output::values::{dict, enum_value, items}; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; - -/// `render(value=value)`. -fn render<'py>(value: Bound<'py, PyAny>) -> PyResult> { - Ok(PyString::new(value.py(), &render_value(&value)?).into_any()) -} - -/// `render(value=value)` when set, else `None`. -fn optional_render<'py>(value: Bound<'py, PyAny>) -> PyResult> { - if value.is_none() { - return Ok(value); - } - render(value) -} - -fn bounds<'py>( - evidence: &Bound<'py, PyAny>, - (start, end): (&str, &str), -) -> PyResult> { - Ok(dict( - evidence.py(), - vec![ - ("start", render(native_field(evidence, start)?)?), - ("end", render(native_field(evidence, end)?)?), - ], - )? - .into_any()) -} - -/// Cap policy, bounds and physical input evidence of future-cursor safety, or `None`. -pub(crate) fn future_cursor_safety<'py>( - evidence: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = evidence.py(); - if evidence.is_none() { - return Ok(evidence.clone()); - } - let relation: Bound<'py, PyAny> = native_field(evidence, "determining_relation")?; - let determining: Bound<'py, PyAny> = if relation.is_none() { - relation - } else { - dict( - py, - vec![ - ("relation", relation), - ( - "cursor_column", - native_field(evidence, "determining_cursor_column")?, - ), - ], - )? - .into_any() - }; - let mut inputs: Vec> = Vec::new(); - for item in items(&native_field(evidence, "inputs")?)? { - inputs.push(dict( - py, - vec![ - ("relation", native_field(&item, "relation")?), - ("cursor_column", native_field(&item, "cursor_column")?), - ("minimum", optional_render(native_field(&item, "minimum")?)?), - ("maximum", render(native_field(&item, "maximum")?)?), - ], - )?); - } - Ok(dict( - py, - vec![ - ("action", enum_value(evidence, "action")?), - ("max_distance", native_field(evidence, "max_distance")?), - ( - "invocation_time", - render(native_field(evidence, "invocation_time")?)?, - ), - ( - "discovered_bounds", - bounds(evidence, ("discovered_start", "discovered_end"))?, - ), - ( - "applied_bounds", - bounds(evidence, ("applied_start", "applied_end"))?, - ), - ( - "maximum_allowed_bounds", - bounds(evidence, ("maximum_allowed_start", "maximum_allowed_end"))?, - ), - ( - "future_start_detected", - native_field(evidence, "future_start_detected")?, - ), - ( - "future_end_detected", - native_field(evidence, "future_end_detected")?, - ), - ("determining_input", determining), - ("inputs", pyo3::types::PyList::new(py, inputs)?.into_any()), - ], - )? - .into_any()) -} - -/// Automatic-start policy and physical eligibility evidence, or `None`. -pub(crate) fn maximum_start_safety<'py>( - evidence: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = evidence.py(); - if evidence.is_none() { - return Ok(evidence.clone()); - } - Ok(dict( - py, - vec![ - ("action", enum_value(evidence, "action")?), - ("max_ahead", native_field(evidence, "max_ahead")?), - ( - "invocation_time", - render(native_field(evidence, "invocation_time")?)?, - ), - ( - "physical_target_max", - render(native_field(evidence, "physical_target_max")?)?, - ), - ( - "highest_eligible_target_max", - optional_render(native_field(evidence, "highest_eligible_target_max")?)?, - ), - ( - "effective_start", - render(native_field(evidence, "effective_start")?)?, - ), - ( - "maximum_allowed_start", - render(native_field(evidence, "maximum_allowed_start")?)?, - ), - ( - "input", - dict( - py, - vec![ - ("relation", native_field(evidence, "target_relation")?), - ("cursor_column", native_field(evidence, "cursor_column")?), - ], - )? - .into_any(), - ), - ], - )? - .into_any()) -} - -/// Serialize maximum automatic-start safety evidence; `None` stays `None`. -#[pyfunction] -fn serialize_maximum_start_safety<'py>(evidence: Bound<'py, PyAny>) -> PyResult> { - maximum_start_safety(&evidence) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(serialize_maximum_start_safety, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/mod.rs index 301a6f231c..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/mod.rs @@ -1,6 +1 @@ -pub(crate) mod actions; -pub(crate) mod detect; -pub(crate) mod metadata; pub(crate) mod registration; -pub(crate) mod renames; -pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/registration.rs index 750b787720..84853c8e28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/registration.rs @@ -2,11 +2,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::planner_changes::{actions, detect, metadata, renames}; +use sqlbuild_python_planner::planner_changes::main::register_actions::register_actions; +use sqlbuild_python_planner::planner_changes::main::register_detect::register_detect; +use sqlbuild_python_planner::planner_changes::main::register_metadata::register_metadata; +use sqlbuild_python_planner::planner_changes::main::register_renames::register_renames; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - actions::register(module)?; - detect::register(module)?; - metadata::register(module)?; - renames::register(module) + register_actions(module)?; + register_detect(module)?; + register_metadata(module)?; + register_renames(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs index 584c84460b..dc54b70121 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -1,15 +1,3 @@ //! Plan entry and plan output assembly over native planner resolution. -pub(crate) mod clone_plans; -pub(crate) mod command_plans; -pub(crate) mod contexts; -pub(crate) mod display_plan; -pub(crate) mod execution_plan; -pub(crate) mod getters; -pub(crate) mod model_entries; -pub(crate) mod models; -pub(crate) mod plan_output; pub(crate) mod registration; -pub(crate) mod relations; -mod test_planning; -mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs deleted file mode 100644 index e29ce386a0..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs +++ /dev/null @@ -1,99 +0,0 @@ -//! Typed requests of plan entry construction. - -use pyo3::prelude::{Bound, Py, PyAny}; -use pyo3::pyclass; -use sqlbuild_python_project::native_host::models::ModelLookup; - -/// What resolving relation locations and source metadata for one scope reads. -#[derive(Debug)] -pub(crate) struct RelationsRequest<'py> { - pub(crate) project: Bound<'py, PyAny>, - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) connection: Bound<'py, PyAny>, - pub(crate) scope: Bound<'py, PyAny>, - pub(crate) deferral: Bound<'py, PyAny>, - pub(crate) project_config: Bound<'py, PyAny>, - pub(crate) local_config: Bound<'py, PyAny>, - pub(crate) known_source_columns: Bound<'py, PyAny>, - pub(crate) relation_keys: Bound<'py, PyAny>, -} - -/// What building model plan entries for one execution scope reads. -#[derive(Debug)] -pub(crate) struct PlanEntriesRequest<'py> { - pub(crate) project: Bound<'py, PyAny>, - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) scope: Bound<'py, PyAny>, - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) relations: Bound<'py, PyAny>, - pub(crate) resolved_models: Bound<'py, PyAny>, - pub(crate) cursor_overrides: Bound<'py, PyAny>, - pub(crate) full_refresh: bool, - pub(crate) build_inputs: Bound<'py, PyAny>, - /// The effective worker limit execution will use. - pub(crate) concurrency: i64, -} - -/// What assembling the plan output's fields reads. -#[derive(Debug)] -pub(crate) struct PlanOutputInputs<'py> { - pub(crate) project: Bound<'py, PyAny>, - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) scope: Bound<'py, PyAny>, - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) relations: Bound<'py, PyAny>, - pub(crate) changes: Bound<'py, PyAny>, - pub(crate) model_entries: Vec>, - pub(crate) model_warnings: Vec>, - pub(crate) reload_sources: Bound<'py, PyAny>, - pub(crate) extras: Bound<'py, PyAny>, -} - -/// Resolved relation and source inputs for plan entry construction. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct PlannerRelationsContext { - pub(crate) model_locations: Py, - pub(crate) seed_locations: Py, - pub(crate) function_locations: Py, - pub(crate) source_map: Py, - pub(crate) source_read_map: Py, - pub(crate) python_source_read_map: Py, - pub(crate) source_warehouse_columns: Py, - pub(crate) star_exclude_keyword: Py, - pub(crate) listed_source_names: Py, -} - -/// Relation lookups and source columns for building one model plan entry. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct ModelPlanContext { - pub(crate) model_locations: Py, - pub(crate) models_by_name: ModelLookup, - pub(crate) seed_locations: Py, - pub(crate) function_locations: Py, - pub(crate) source_map: Py, - pub(crate) source_warehouse_columns: Py, - pub(crate) star_exclude_keyword: Py, - pub(crate) functions_by_name: Py, - pub(crate) runtime_cursor_producer_names: Py, - pub(crate) future_cursor_config: Py, - pub(crate) start_cursor_config: Py, - pub(crate) invocation_time: Py, -} - -/// Blocked models and cursor overrides for plan entry building. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct PlanEntryBuildInputs { - pub(crate) source_freshness_blocked_model_names: Py, - pub(crate) external_blocked_model_names: Py, - pub(crate) start_cursor_override: Py, - pub(crate) end_cursor_override: Py, - pub(crate) future_cursor_config: Py, - pub(crate) start_cursor_config: Py, - pub(crate) invocation_time: Py, - pub(crate) max_microbatches: Py, - pub(crate) max_microbatches_is_override: bool, - pub(crate) microbatch_limit_action: Py, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs index ed45e8827f..174bc7b89f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs @@ -1,17 +1,26 @@ //! Register plan entry and plan output assembly entry points. use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use sqlbuild_python_planner::planner_entries::main::entries::register_model_entries::register_model_entries; +use sqlbuild_python_planner::planner_entries::main::entries::register_relations::register_relations; +use sqlbuild_python_planner::planner_entries::main::plans::register_clone_plans::register_clone_plans; +use sqlbuild_python_planner::planner_entries::main::plans::register_command_plans::register_command_plans; +use sqlbuild_python_planner::planner_entries::main::plans::register_display_plan::register_display_plan; +use sqlbuild_python_planner::planner_entries::main::plans::register_execution_plan::register_execution_plan; +use sqlbuild_python_planner::planner_entries::main::plans::register_plan_output::register_plan_output; +use sqlbuild_python_planner::planner_entries::main::plans::register_test_planning::register_test_planning; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - super::clone_plans::register(module)?; - super::command_plans::register(module)?; - super::display_plan::register(module)?; - super::execution_plan::register(module)?; - super::model_entries::register(module)?; - super::plan_output::register(module)?; - super::relations::register(module)?; - super::test_planning::register(module) + module + .add_class::()?; + module.add_class::()?; + module.add_class::()?; + register_clone_plans(module)?; + register_command_plans(module)?; + register_display_plan(module)?; + register_execution_plan(module)?; + register_model_entries(module)?; + register_plan_output(module)?; + register_relations(module)?; + register_test_planning(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs index f8f6364a35..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs @@ -1,4 +1 @@ -pub(crate) mod identities; -pub(crate) mod metadata; pub(crate) mod registration; -pub(crate) mod staleness; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs index cce0b3f494..4dd3a32767 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs @@ -2,14 +2,16 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::planner_identity::{identities, metadata, staleness}; +use sqlbuild_python_planner::planner_identity::main::register_identities::register_identities; +use sqlbuild_python_planner::planner_identity::main::register_metadata::register_metadata; +use sqlbuild_python_planner::planner_identity::main::register_staleness::register_staleness; use sqlbuild_python_project::planner_identity::main::register_graph::register_graph; use sqlbuild_python_project::planner_identity::main::register_hashing::register_hashing; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { register_graph(module)?; register_hashing(module)?; - identities::register(module)?; - metadata::register(module)?; - staleness::register(module) + register_identities(module)?; + register_metadata(module)?; + register_staleness(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs deleted file mode 100644 index 09e5891b19..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs +++ /dev/null @@ -1,14 +0,0 @@ -use sqlbuild_host::entries::models::AdapterHost; -pub(super) const AUTOMATIC: &str = "automatic"; -pub(super) const DONE: &str = "done"; -pub(super) const PENDING_ARCHIVE: &str = "pending_archive"; -pub(super) const PENDING_VIEW: &str = "pending_view"; -pub(super) const LIST_RELATIONS_FOR_INSPECTION: AdapterHost = - AdapterHost::ListRelationsForInspection; -pub(super) const GET_COLUMNS_FOR_INSPECTION: AdapterHost = AdapterHost::GetColumnsForInspection; -pub(super) const QUALIFIED_RELATION_MAX_PARTS: usize = 3; -pub(super) const MIGRATION_MODEL_NAME_METADATA_KEY: &str = "model_name"; -pub(super) const HIDDEN_ORIGIN_REMEDY: &str = "the connection may lack permission to see it, or it was dropped outside SQLBuild; restore access to it, or remove migrate_from to build the model fresh deliberately (missing_migration_origin does not apply)"; -pub(super) const MISSING_ORIGIN_ALLOW_OUTCOME: &str = "nothing is migrated and the build continues as if migrate_from were absent; remove migrate_from once no target still needs it"; -pub(super) const MISSING_ORIGIN_CONFIRMATION_OUTCOME: &str = "building without it needs confirmation (missing_migration_origin = require_confirmation); pass --allow-missing-migration-origin, or remove migrate_from if it is no longer needed"; -pub(super) const MISSING_ORIGIN_DENY_OUTCOME: &str = "the build stops (missing_migration_origin = deny); if the migration already happened elsewhere or is no longer needed, remove migrate_from"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs deleted file mode 100644 index 53d4a0e078..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs +++ /dev/null @@ -1,118 +0,0 @@ -//! Migration identity binding with unchanged metadata and hash format. - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::planner_migrations::main::{migration_fingerprint, migration_ref_names}; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; -use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; -use sqlbuild_python_project::native_host::models::ModelLookup; -use std::collections::BTreeMap; - -/// Hash normalized SQL and non-storage metadata through rename identities. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn build_migration_fingerprint( - py: Python<'_>, - options: Option>, -) -> PyResult> { - let arguments = KeywordArguments::read( - py, - "build_migration_fingerprint", - options, - ( - &["query_sql", "metadata_json", "ref_identities", "dialect"], - &[], - ), - )?; - let mut identities: BTreeMap = BTreeMap::new(); - for item in arguments - .value("ref_identities")? - .call_method0("items")? - .try_iter()? - { - let (key, value): (String, String) = item?.extract()?; - identities.insert(key, value); - } - let dialect: Option = arguments.value("dialect")?.extract()?; - fingerprint( - &arguments.value("query_sql")?.extract::()?, - &arguments.value("metadata_json")?.extract::()?, - &identities, - dialect.as_deref(), - ) -} - -/// The migration fingerprint of SQL and metadata through rename identities, `generic` by default. -pub(crate) fn fingerprint( - sql: &str, - metadata: &str, - identities: &BTreeMap, - dialect: Option<&str>, -) -> PyResult> { - migration_fingerprint::migration_fingerprint( - sql, - metadata, - identities, - dialect - .filter(|value| !value.is_empty()) - .unwrap_or("generic"), - ) - .map_err(PyValueError::new_err) -} - -/// Every name whose rename identity can change the migration fingerprint. -pub(crate) fn migration_fingerprint_ref_names<'py>( - py: Python<'py>, - query_sql: &str, - metadata_json: &str, -) -> PyResult> { - PyFrozenSet::new( - py, - migration_ref_names::migration_ref_names(query_sql, metadata_json) - .map_err(PyValueError::new_err)?, - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_migration_fingerprint, module)?)?; - Ok(()) -} - -/// Attach persisted migration identities through the existing cache boundary. -pub(crate) fn with_migration_fingerprints<'py>( - entries: &Bound<'py, PyAny>, - models: &ModelLookup, - cache: &Bound<'py, PyAny>, - dialect: &Bound<'py, PyAny>, -) -> PyResult> { - use sqlbuild_python_base::boundary::main::call_with::call_with; - let py = entries.py(); - let mut attached: Vec> = Vec::new(); - for entry in entries.try_iter()? { - let entry = entry?; - let model = models.get(py, &native_field(&entry, "name")?.extract::()?)?; - let metadata = native_field(&entry, "fingerprint_metadata_json")?; - if model.is_none() || metadata.is_none() || native_field(&entry, "action")?.eq("skip")? { - attached.push(entry); - continue; - } - let fingerprint = call_with( - &native_field(cache, "fingerprint")?, - &[ - ("query_sql", query_sql(&model)?), - ("metadata_json", metadata), - ("ref_identities", PyDict::new(py).into_any()), - ("dialect", dialect.clone()), - ], - )?; - let kwargs = PyDict::new(py); - kwargs.set_item("migration_fingerprint", fingerprint)?; - attached.push(replaced_keywords(&entry, &kwargs)?); - } - cache.call_method0("persist")?; - PyTuple::new(py, attached) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index b5361f630c..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -1,35 +1 @@ -mod column_candidates; -mod column_entries; -mod column_objects; -mod column_overlay; -pub(crate) mod column_planning; -mod column_warnings; -pub(crate) mod columns; -mod compatibility; -mod constants; -mod discovery; -mod discovery_events; -mod discovery_models; -mod discovery_requests; -mod errors; -pub(crate) mod fingerprint; -pub(crate) mod getters; -mod known_input_columns; -mod matching; -mod model_entries; -mod model_handover; -mod model_overlay; -pub(crate) mod model_planning; -mod model_warnings; -pub(crate) mod models; -mod old_name_entries; -mod old_name_objects; -pub(crate) mod old_name_planning; -mod old_name_references; -pub(crate) mod old_names; -mod planning_boundary; -mod planning_execution; -mod planning_origin; -pub(crate) mod projections; pub(crate) mod registration; -pub(crate) mod state_inspection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs deleted file mode 100644 index 4d04495439..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs +++ /dev/null @@ -1,150 +0,0 @@ -use pyo3::prelude::{Bound, PyAny}; -use pyo3::types::PyDict; - -#[derive(Debug)] -pub(crate) struct OldNamePlanner<'py> { - pub runtime: Bound<'py, PyAny>, - pub models: Vec>, - pub histories: Vec>, - pub views: Bound<'py, PyAny>, - pub columns: Bound<'py, PyAny>, - pub retention: Bound<'py, PyAny>, - pub now: Bound<'py, PyAny>, -} - -#[derive(Debug)] -pub(crate) struct ColumnCandidate<'py> { - pub model: Bound<'py, PyAny>, - pub warehouse: Bound<'py, PyDict>, - pub declared: Bound<'py, PyDict>, - pub fingerprint: Bound<'py, PyAny>, - pub previous: Bound<'py, PyAny>, - pub current: Bound<'py, PyAny>, -} -#[derive(Debug)] -pub(crate) struct CheckedColumns<'py> { - pub entries: Vec>, - pub warnings: Vec>, -} - -/// The planning-phase inputs every migration planner reads. -#[derive(Debug)] -pub(crate) struct MigrationPhase<'py> { - pub(crate) runtime: Bound<'py, PyAny>, - pub(crate) scope: Bound<'py, PyAny>, - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) overrides: Bound<'py, PyAny>, - pub(crate) deferral: Bound<'py, PyAny>, -} - -/// The column migration planning inputs beyond the phase. -#[derive(Debug)] -pub(crate) struct ColumnMigrationInputs<'py> { - pub(crate) full_refresh_model_names: Bound<'py, PyAny>, - pub(crate) physical_relations: Bound<'py, PyAny>, - pub(crate) source_columns: Bound<'py, PyAny>, -} - -/// The snapshot after model migrations overlay it, with the planned entries and their warnings. -#[derive(Debug)] -pub(crate) struct ModelMigrationPlanning<'py> { - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) entries: Vec>, - pub(crate) warnings: Vec>, -} - -/// The snapshot after column renames overlay it, with the planned entries, rename hints and warnings. -#[derive(Debug)] -pub(crate) struct ColumnMigrationPlanning<'py> { - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) entries: Vec>, - pub(crate) hints: Vec>, - pub(crate) warnings: Vec>, -} - -/// The old-name view steps, the recorded live views and the warnings they raise. -#[derive(Debug)] -pub(crate) struct OldNameViewPlanning<'py> { - pub(crate) entries: Bound<'py, PyAny>, - pub(crate) views: Bound<'py, PyAny>, - pub(crate) warnings: Vec>, -} - -/// One destination model whose data should come from an earlier relation. -#[derive(Debug, Clone)] -pub(crate) struct ModelMigrationRequest<'py> { - pub(crate) model: Bound<'py, PyAny>, - pub(crate) discovery: Bound<'py, PyAny>, - pub(crate) raw_origin: Option>, - pub(crate) origin_location: Option>, - pub(crate) origin_model: Bound<'py, PyAny>, - pub(crate) force: bool, - pub(crate) identity_only: bool, -} - -/// One project model's explicit migrate_from, resolved to a relation. -#[derive(Debug)] -pub(crate) struct ModelMigrationDeclaration<'py> { - pub(crate) model_name: Bound<'py, PyAny>, - pub(crate) origin_location: Bound<'py, PyAny>, - pub(crate) origin_model: Bound<'py, PyAny>, -} - -/// Manual and automatically discovered migration requests for one plan. -#[derive(Debug)] -pub(crate) struct ModelMigrationDiscovery<'py> { - pub(crate) requests: Vec>, - pub(crate) warnings: Vec>, - pub(crate) destination_fingerprints: Bound<'py, PyDict>, -} - -/// Compatibility status plus human-readable blocking findings. -#[derive(Debug)] -pub(crate) struct MigrationCompatibilityResult<'py> { - pub(crate) status: Bound<'py, PyAny>, - pub(crate) findings: Bound<'py, PyAny>, -} - -/// The fingerprint inputs shared by migration matching and destination fingerprints. -#[derive(Debug)] -pub(crate) struct FingerprintInputs<'py> { - pub(crate) metadata_jsons: Bound<'py, PyAny>, - pub(crate) dialect: Bound<'py, PyAny>, - pub(crate) fingerprints: Bound<'py, PyAny>, -} - -/// One resolved migration request decided against live state. -#[derive(Debug)] -pub(crate) struct MigrationDecisionRequest<'a, 'py> { - pub(crate) runtime: Bound<'py, PyAny>, - pub(crate) request: &'a ModelMigrationRequest<'py>, - pub(crate) origin: Bound<'py, PyAny>, - pub(crate) origin_model: Bound<'py, PyAny>, - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) state: Bound<'py, PyAny>, - pub(crate) snapshots_config: Bound<'py, PyAny>, -} - -/// What checking an origin relation's compatibility with its destination model reads. -#[derive(Debug)] -pub(crate) struct CompatibilityCheck<'py> { - pub(crate) model: Bound<'py, PyAny>, - pub(crate) origin_relation: Bound<'py, PyAny>, - pub(crate) origin_columns: Bound<'py, PyAny>, - pub(crate) sql_analysis_enabled: Bound<'py, PyAny>, - pub(crate) column_dialect: Bound<'py, PyAny>, - pub(crate) snapshots_config: Bound<'py, PyAny>, -} - -/// What discovering automatic and manual migration requests reads. -#[derive(Debug)] -pub(crate) struct DiscoveryRequest<'py> { - pub(crate) runtime: Bound<'py, PyAny>, - pub(crate) scope: Bound<'py, PyAny>, - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) manual_requests: Vec>, - pub(crate) declarations: Vec>, - pub(crate) state: Bound<'py, PyAny>, - pub(crate) project_schemas: Bound<'py, PyAny>, - pub(crate) fingerprints: Bound<'py, PyAny>, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs index 8e819c060c..1d370fd5a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs @@ -1,11 +1,17 @@ //! Register pure migration helpers. use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use sqlbuild_python_planner::planner_migrations::main::register_column_planning::register_column_planning; +use sqlbuild_python_planner::planner_migrations::main::register_columns::register_columns; +use sqlbuild_python_planner::planner_migrations::main::register_fingerprint::register_fingerprint; +use sqlbuild_python_planner::planner_migrations::main::register_projections::register_projections; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - super::columns::register(module)?; - super::column_planning::register(module)?; - super::projections::register(module)?; - module.add_class::()?; - super::fingerprint::register(module) + register_columns(module)?; + register_column_planning(module)?; + register_projections(module)?; + module + .add_class::( + )?; + register_fingerprint(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs deleted file mode 100644 index 5c45d15791..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! Python modules and names the planner phases exchange values with. - -pub(crate) const SNOWFLAKE_ADAPTER: &str = "snowflake"; -pub(crate) const BIGQUERY_ADAPTER: &str = "bigquery"; -pub(crate) const TRANSIENT_TABLE_TYPE: &str = "transient"; -pub(crate) const PERMANENT_TABLE_TYPE: &str = "permanent"; -pub(crate) const DECREASE_WARNING: &str = - "Retention decreases are irreversible once warehouse history expires."; -pub(crate) const DOWNGRADE_HISTORY_DAYS: i64 = 90; -pub(crate) const FULL_REFRESH_DISABLED_REBUILD_CODE: &str = "S203"; -pub(crate) const MISSING_UPSTREAM_CODE: &str = "S301"; -pub(crate) const RETENTION_DECREASE: &str = "decrease"; -pub(crate) const RETENTION_INCREASE: &str = "increase"; -pub(crate) const RETENTION_MATCH: &str = "match"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs deleted file mode 100644 index c009111ea3..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Planner scope and identity contexts: built from keywords and read by planner phases. - -use pyo3::prelude::{Bound, Py, PyAny, PyResult}; -use pyo3::pymethods; -use pyo3::{PyClass, pyclass::boolean_struct::True}; - -use crate::bindings::_helpers::planner_phases::models::{ - PlannerIdentityContext, PlannerScopeResolution, -}; -use sqlbuild_python_project::planner_scope::main::project_scope::project_scope; - -#[pymethods] -impl PlannerScopeResolution { - #[new] - #[pyo3(signature = (*, selected_scope, stale_warning_scope, inspection_scope))] - fn new( - selected_scope: Py, - stale_warning_scope: Py, - inspection_scope: Py, - ) -> Self { - Self { - selected_scope, - stale_warning_scope, - inspection_scope, - } - } - - #[getter] - fn selected_scope<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { - project_scope(self.selected_scope.bind(py)) - } - - #[getter] - fn stale_warning_scope<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { - project_scope(self.stale_warning_scope.bind(py)) - } - - #[getter] - fn inspection_scope<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { - project_scope(self.inspection_scope.bind(py)) - } -} - -#[pymethods] -impl PlannerIdentityContext { - #[new] - #[pyo3(signature = (*, version_identities, stale_warning_identities))] - fn new(version_identities: Py, stale_warning_identities: Py) -> Self { - Self { - version_identities, - stale_warning_identities, - } - } -} - -/// One field of a frozen native planner class passed as a Python object. -pub(crate) fn class_field<'py, T>( - value: &Bound<'py, PyAny>, - field: fn(&T) -> &Py, -) -> PyResult> -where - T: PyClass + Sync, -{ - Ok(field(value.cast::()?.get()).bind(value.py()).clone()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs deleted file mode 100644 index ef44a32502..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs +++ /dev/null @@ -1,98 +0,0 @@ -//! Per-model full-refresh resolution and full-rebuild refusals. - -use crate::bindings::_helpers::planner_phases::constants::FULL_REFRESH_DISABLED_REBUILD_CODE; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::pyfunction; -use pyo3::types::PyBool; -use sqlbuild_analysis::planner_warehouse::main::full_rebuild_cause::full_rebuild_cause; -use sqlbuild_analysis::planner_warehouse::models::FullRebuildFacts; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::model_config::main::config::config_value::config_value; -use sqlbuild_python_front::model_config::main::config::model_values::model_values; -use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; -use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; -use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; -use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; -use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; -use sqlbuild_python_project::planner_phases::main::strings::strings; - -const FULL_REFRESH_KEY: &str = "full_refresh"; - -fn configured_full_refresh<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - config_value(model.py(), &model_values(model)?, FULL_REFRESH_KEY) -} - -/// dbt-compatible nullable model full-refresh semantics. -pub(crate) fn resolve_model_full_refresh( - model: &Bound<'_, PyAny>, - cli_full_refresh: bool, -) -> PyResult { - let configured = configured_full_refresh(model)?; - if configured.is_instance_of::() { - return configured.extract(); - } - Ok(cli_full_refresh) -} - -/// Names of models effectively full refreshed for this invocation. -pub(crate) fn effectively_full_refreshed_model_names<'py>( - project: &Bound<'py, PyAny>, - cli_full_refresh: bool, -) -> PyResult> { - let mut names: Vec<_> = Vec::new(); - for model in compiled_models(project)?.try_iter()? { - let model = model?; - if resolve_model_full_refresh(&model, cli_full_refresh)? { - names.push(name_object(&model)?); - } - } - frozenset(project.py(), &names) -} - -#[pyfunction] -#[pyo3(signature = (*, change_result, full_refresh))] -pub(crate) fn describe_full_rebuild_cause( - change_result: &Bound<'_, PyAny>, - full_refresh: bool, -) -> PyResult { - Ok(full_rebuild_cause(&FullRebuildFacts { - full_refresh, - change_kind: native_field(change_result, "change_kind")? - .str()? - .extract::()?, - query_changed: native_field(change_result, "query_changed")?.is_truthy()?, - changed_functions: strings(&native_field(change_result, "changed_functions")?)?, - has_schema_findings: native_field(change_result, "schema_findings")?.is_truthy()?, - })) -} - -#[pyfunction] -#[pyo3(signature = (*, model, change_result, full_rebuild_cause))] -pub(crate) fn check_full_rebuild_allowed( - model: &Bound<'_, PyAny>, - change_result: &Bound<'_, PyAny>, - full_rebuild_cause: &str, -) -> PyResult<()> { - let py = model.py(); - if !configured_full_refresh(model)?.is(PyBool::new(py, false)) { - return Ok(()); - } - if native_field(change_result, "change_kind")?.eq("first_run")? - || native_field(change_result, "query_changed")?.is_truthy()? - { - return Ok(()); - } - let name: String = model_name(model)?; - Err(planner_error( - py, - &format!( - "model '{name}' sets full_refresh false, but the plan would fully rebuild it ({full_rebuild_cause})" - ), - ( - Some(FULL_REFRESH_DISABLED_REBUILD_CODE), - Some( - "set replay_on_change to forward or bounded- on this model to keep its existing history, or remove full_refresh false to allow the rebuild", - ), - ), - )) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs index c952ce457a..b84ad98904 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs @@ -1,17 +1,3 @@ //! Native execution-planning phases over Python planner models and adapter callbacks. -pub(crate) mod buildability; -pub(crate) mod constants; -pub(crate) mod contexts; -pub(crate) mod deferred_origins; -pub(crate) mod execution_changes; -pub(crate) mod full_refresh; -pub(crate) mod getters; -pub(crate) mod identities; -pub(crate) mod models; -pub(crate) mod output_assembly; pub(crate) mod registration; -pub(crate) mod retention; -pub(crate) mod runtime; -pub(crate) mod scopes; -pub(crate) mod warehouse_state; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs deleted file mode 100644 index fc33a2e8b3..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs +++ /dev/null @@ -1,116 +0,0 @@ -//! Plan output fields gathered across the plan phases before the output is built once. - -use std::collections::BTreeMap; - -use pyo3::prelude::{Bound, Py, PyAny}; -use pyo3::pyclass; - -/// The plan output's fields and metadata entries, each set before the output is built. -#[derive(Debug)] -pub(crate) struct PlanOutputFields<'py> { - pub(crate) fields: Vec<(&'static str, Bound<'py, PyAny>)>, - pub(crate) metadata: Vec<(&'static str, Bound<'py, PyAny>)>, -} - -/// The model changes and resolved actions after write-hash reconciliation. -#[derive(Debug)] -pub(crate) struct ChangeReconciliation<'py> { - pub(crate) changes: Bound<'py, PyAny>, - pub(crate) resolved_actions: Bound<'py, PyAny>, -} - -/// Planned model entries and the warnings planning them raised. -#[derive(Debug)] -pub(crate) struct ModelEntryResults<'py> { - pub(crate) entries: Vec>, - pub(crate) warnings: Vec>, -} - -/// The runtime, warehouse state, identities and overrides the base plan output reads. -#[derive(Debug)] -pub(crate) struct PlanOutputContext<'a, 'py> { - pub(crate) runtime: &'a Bound<'py, PyAny>, - pub(crate) warehouse: &'a Bound<'py, PyAny>, - pub(crate) identities: &'a Bound<'py, PyAny>, - pub(crate) overrides: &'a Bound<'py, PyAny>, -} - -/// The execution scope, reconciled changes, planned entries and source freshness of a plan. -#[derive(Debug)] -pub(crate) struct PlannedModelEntries<'a, 'py> { - pub(crate) execution_scope: &'a Bound<'py, PyAny>, - pub(crate) reconciliation: &'a ChangeReconciliation<'py>, - pub(crate) entries: ModelEntryResults<'py>, - pub(crate) source_freshness: &'a Bound<'py, PyAny>, -} - -/// The runtime, warehouse state, scopes and policies storage policy planning reads. -#[derive(Debug)] -pub(crate) struct StoragePolicyContext<'a, 'py> { - pub(crate) runtime: &'a Bound<'py, PyAny>, - pub(crate) warehouse: &'a Bound<'py, PyAny>, - pub(crate) scopes: &'a Bound<'py, PyAny>, - pub(crate) policies: &'a Bound<'py, PyAny>, -} - -/// The scopes, warehouse state, identities and changes stale-selection warnings read. -#[derive(Debug)] -pub(crate) struct PlanWarningContext<'a, 'py> { - pub(crate) scopes: &'a Bound<'py, PyAny>, - pub(crate) warehouse: &'a Bound<'py, PyAny>, - pub(crate) identities: &'a Bound<'py, PyAny>, - pub(crate) stale_warning_changes: &'a Bound<'py, PyAny>, -} - -/// Resolved execution environment shared by planner phases. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct PlannerRuntime { - pub(crate) project: Py, - pub(crate) adapter: Py, - pub(crate) connection: Py, - pub(crate) project_config: Py, - pub(crate) local_config: Py, - pub(crate) on_progress: Py, - pub(crate) invocation_time: Py, -} - -/// Warehouse snapshot, inspection relations and migration plans gathered once per plan. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct PlannerWarehouseState { - pub(crate) snapshot: Py, - pub(crate) inspection_relations: Py, - pub(crate) migration_entries: Py, - pub(crate) migration_warnings: Py, - pub(crate) column_migration_entries: Py, - pub(crate) column_rename_hints: Py, - pub(crate) old_name_view_entries: Py, - pub(crate) old_name_views: Py, - pub(crate) migration_fingerprints: Py, -} - -/// Resolved planner scopes for one plan build. -#[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug)] -pub(crate) struct PlannerScopeResolution { - pub(crate) selected_scope: Py, - pub(crate) stale_warning_scope: Py, - pub(crate) inspection_scope: Py, -} - -/// Expected version identities for inspection and stale-warning scopes. -#[pyclass(module = "sqlbuild._native", frozen, get_all)] -#[derive(Debug)] -pub(crate) struct PlannerIdentityContext { - pub(crate) version_identities: Py, - pub(crate) stale_warning_identities: Py, -} - -/// Planner identity work since the last reset, for work-count tests. -#[pyclass] -#[derive(Debug, Default)] -pub(crate) struct IdentityWork { - pub(crate) identity_builds: u64, - pub(crate) stale_detections: BTreeMap, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs index 5a57fe9c6b..6f51aa9d97 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -1,47 +1,28 @@ //! Register the native execution-planning phases. use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::wrap_pyfunction; +use sqlbuild_python_project::planner_scope::main::resolve_planner_scopes_function::resolve_planner_scopes_function; +use sqlbuild_python_planner::planner_phases::main::outputs::detect_stale_warning_changes_of_function::detect_stale_warning_changes_of_function; +use sqlbuild_python_planner::planner_phases::main::outputs::planner_identity_work_info_function::planner_identity_work_info_function; +use sqlbuild_python_planner::planner_phases::main::rebuilds::check_full_rebuild_allowed_function::check_full_rebuild_allowed_function; +use sqlbuild_python_planner::planner_phases::main::rebuilds::describe_full_rebuild_cause_function::describe_full_rebuild_cause_function; +use sqlbuild_python_planner::planner_phases::main::rebuilds::plan_retention_function::plan_retention_function; +use sqlbuild_python_planner::planner_phases::main::rebuilds::plan_table_types_function::plan_table_types_function; +use sqlbuild_python_planner::planner_phases::main::rebuilds::table_type_copy_name_function::table_type_copy_name_function; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; module - .add_class::()?; - module - .add_class::()?; - module - .add_class::()?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::identities::planner_identity_work_info, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::identities::detect_stale_warning_changes_of, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::full_refresh::describe_full_rebuild_cause, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::full_refresh::check_full_rebuild_allowed, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::retention::table_type_copy_name, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::retention::plan_table_types, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_phases::retention::plan_retention, - module - )?)?; + .add_class::()?; + module.add_function(resolve_planner_scopes_function(module)?)?; + module.add_function(planner_identity_work_info_function(module)?)?; + module.add_function(detect_stale_warning_changes_of_function(module)?)?; + module.add_function(describe_full_rebuild_cause_function(module)?)?; + module.add_function(check_full_rebuild_allowed_function(module)?)?; + module.add_function(table_type_copy_name_function(module)?)?; + module.add_function(plan_table_types_function(module)?)?; + module.add_function(plan_retention_function(module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/constants.rs deleted file mode 100644 index b443c6acb1..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/constants.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! Planner cursor configuration keys. - -pub(super) const CURSOR_START: &str = "cursor_start"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs deleted file mode 100644 index 01961f6424..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs +++ /dev/null @@ -1,41 +0,0 @@ -//! Location maps retaining the original compiled Python destination objects. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_python_project::resource_views::main::resource_reads::destination::destination; -use sqlbuild_python_project::resource_views::main::resource_reads::name::name; - -fn locations<'py>(items: &Bound<'py, PyAny>) -> PyResult> { - let result = PyDict::new(items.py()); - for item in items.try_iter()? { - let item = item?; - result.set_item(name(&item)?, destination(&item)?)?; - } - Ok(result) -} - -#[pyfunction] -pub(crate) fn build_model_locations<'py>( - models: &Bound<'py, PyAny>, -) -> PyResult> { - locations(models) -} - -#[pyfunction] -pub(crate) fn build_seed_locations<'py>(seeds: &Bound<'py, PyAny>) -> PyResult> { - locations(seeds) -} - -#[pyfunction] -pub(crate) fn build_function_locations<'py>( - functions: &Bound<'py, PyAny>, -) -> PyResult> { - locations(functions) -} - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_model_locations, module)?)?; - module.add_function(wrap_pyfunction!(build_seed_locations, module)?)?; - module.add_function(wrap_pyfunction!(build_function_locations, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs index 93bab4d84b..a21268c7ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs @@ -1,20 +1,3 @@ //! Planner reference resolution with Python adapter rendering callbacks. -mod constants; -pub(crate) mod cursor_bounds; -pub(crate) mod cursor_model; -pub(crate) mod cursor_overrides; -pub(crate) mod cursor_policies; -pub(crate) mod cursor_ranges; -mod cursor_safety; -pub(crate) mod cursor_warnings; -pub(crate) mod cursor_windows; -pub(crate) mod future_safety; -pub(crate) mod locations; -pub(crate) mod maximum_safety; -pub(crate) mod models; -pub(crate) mod orchestration; -pub(crate) mod refs; pub(crate) mod registration; -pub(crate) mod sources; -mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs deleted file mode 100644 index db9d2e26f5..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! Typed requests of planner SQL resolution. - -use pyo3::prelude::{Bound, PyAny}; - -/// The cursor window that filters the cursor inputs one resolution reads. -#[derive(Debug)] -pub(crate) struct CursorFiltering<'py> { - pub(crate) bounds: Bound<'py, PyAny>, - pub(crate) filter_inputs: Bound<'py, PyAny>, - pub(crate) cursor_type: Bound<'py, PyAny>, - pub(crate) lower_bound_inclusive: bool, -} - -/// One model's SQL resolution decisions. -#[derive(Debug)] -pub(crate) struct ModelSqlRequest<'py> { - pub(crate) model: Bound<'py, PyAny>, - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) backfill: Bound<'py, PyAny>, - pub(crate) full_refresh: bool, - pub(crate) cursor_overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), - pub(crate) replaces_relation: bool, - pub(crate) full_rebuild_cause: Option, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs index 86ba08ef70..db1babf5ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs @@ -1,17 +1,28 @@ //! Register planner SQL resolution entry points. use pyo3::prelude::{Bound, PyModule, PyResult}; +use sqlbuild_python_planner::planner_resolution::main::cursor::register_cursor_bounds::register_cursor_bounds; +use sqlbuild_python_planner::planner_resolution::main::cursor::register_cursor_overrides::register_cursor_overrides; +use sqlbuild_python_planner::planner_resolution::main::cursor::register_cursor_ranges::register_cursor_ranges; +use sqlbuild_python_planner::planner_resolution::main::cursor::register_cursor_warnings::register_cursor_warnings; +use sqlbuild_python_planner::planner_resolution::main::cursor::register_cursor_windows::register_cursor_windows; +use sqlbuild_python_planner::planner_resolution::main::references::register_locations::register_locations; +use sqlbuild_python_planner::planner_resolution::main::references::register_orchestration::register_orchestration; +use sqlbuild_python_planner::planner_resolution::main::references::register_refs::register_refs; +use sqlbuild_python_planner::planner_resolution::main::references::register_sources::register_sources; +use sqlbuild_python_planner::planner_resolution::main::safety::register_future_safety::register_future_safety; +use sqlbuild_python_planner::planner_resolution::main::safety::register_maximum_safety::register_maximum_safety; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - super::cursor_bounds::register(module)?; - super::cursor_overrides::register(module)?; - super::cursor_ranges::register(module)?; - super::cursor_warnings::register(module)?; - super::future_safety::register(module)?; - super::maximum_safety::register(module)?; - super::cursor_windows::register(module)?; - super::refs::register(module)?; - super::locations::register(module)?; - super::orchestration::register(module)?; - super::sources::register(module) + register_cursor_bounds(module)?; + register_cursor_overrides(module)?; + register_cursor_ranges(module)?; + register_cursor_warnings(module)?; + register_future_safety(module)?; + register_maximum_safety(module)?; + register_cursor_windows(module)?; + register_refs(module)?; + register_locations(module)?; + register_orchestration(module)?; + register_sources(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs deleted file mode 100644 index 230067769f..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! Relation categories with additional scenario binding semantics. - -pub(super) const REF: &str = "ref"; -pub(super) const SOURCE: &str = "source"; -pub(super) const SOURCE_MARKER: usize = 2; -pub(super) const DBT_REF: &str = "dbt_ref"; -pub(super) const SCENARIO_PLAN_INVALID_HASH_PREFIX: &str = "S501"; -pub(super) const SCENARIO_PLAN_HASH_COLLISION: &str = "S502"; -pub(super) const SCENARIO_PLAN_RELATION_COLLISION: &str = "S503"; -pub(super) const SCENARIO_PLAN_GRAPH_VALIDATION: &str = "S504"; -pub(super) const SCENARIO_PLAN_UNKNOWN_SEED: &str = "S507"; -pub(super) const SCENARIO_PLAN_MISSING_FIXTURE_SQL: &str = "S508"; -pub(super) const SCENARIO_PLAN_MISSING_RELATION_TARGET: &str = "S509"; -pub(super) const SCENARIO_PLAN_INVALID_FIXTURE: &str = "S510"; -pub(super) const SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER: &str = "S511"; -pub(super) const SCENARIO_PLAN_EXPECTED_COLUMNS: &str = "S512"; -pub(super) const SCENARIO_EXPECTED_COLUMNS_HELP: &str = "list every column the model outputs in the expected CTE, and remove columns it does not output"; -pub(super) const SCENARIO_PLAN_INTERNAL: &str = "S599"; -/// The zero-argument cursor intrinsics a model query can call. -pub(super) const CURSOR_INTRINSIC_NAMES: [&str; 2] = ["__cursor_start", "__cursor_end"]; -pub(super) const INCREMENTAL_MODE_KEY: &str = "incremental_mode"; -pub(super) const MICROBATCH_INCREMENTAL_MODE: &str = "microbatch"; -pub(super) const CURSOR_TYPE_KEY: &str = "cursor_type"; -pub(super) const INTEGER_CURSOR_TYPE: &str = "integer"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs index 3bacaebca1..43c07cb865 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs @@ -1,15 +1,3 @@ //! Python boundaries for native scenario planning. -pub(crate) mod artifacts; -mod cli; -mod constants; -mod entries; -pub(crate) mod execution; -mod fixtures; -pub(crate) mod graph; -mod helper_scope; -mod models; pub(crate) mod registration; -mod relations; -mod sql; -mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs index c2e450af9b..574dcf8e25 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs @@ -1,50 +1,31 @@ //! Register scenario planning bindings. -use crate::bindings::_helpers::planner_scenario::{ - artifacts, cli, execution, fixtures, graph, relations, sql, -}; +use sqlbuild_python_planner::planner_scenario::main::build_scenario_artifact_name_function::build_scenario_artifact_name_function; +use sqlbuild_python_planner::planner_scenario::main::build_scenario_hash_index_function::build_scenario_hash_index_function; +use sqlbuild_python_planner::planner_scenario::main::build_scenario_relation_map_function::build_scenario_relation_map_function; +use sqlbuild_python_planner::planner_scenario::main::compute_scenario_hash_prefix_function::compute_scenario_hash_prefix_function; +use sqlbuild_python_planner::planner_scenario::main::scenario_artifact_physical_name_check_function::scenario_artifact_physical_name_check_function; +use sqlbuild_python_planner::planner_scenario::main::parse_scenario_artifact_physical_name_function::parse_scenario_artifact_physical_name_function; +use sqlbuild_python_planner::planner_scenario::main::build_cli_scenario_plan_function::build_cli_scenario_plan_function; +use sqlbuild_python_planner::planner_scenario::main::build_scenario_execution_plan_function::build_scenario_execution_plan_function; +use sqlbuild_python_planner::planner_scenario::main::build_scenario_fixture_plans_function::build_scenario_fixture_plans_function; +use sqlbuild_python_planner::planner_scenario::main::plan_scenario_graph_function::plan_scenario_graph_function; +use sqlbuild_python_planner::planner_scenario::main::build_scenario_relation_plan_of_function::build_scenario_relation_plan_of_function; +use sqlbuild_python_planner::planner_scenario::main::resolve_scenario_check_sql_function::resolve_scenario_check_sql_function; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::wrap_pyfunction; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - execution::build_scenario_execution_plan, - module - )?)?; - module.add_function(wrap_pyfunction!( - fixtures::build_scenario_fixture_plans, - module - )?)?; - module.add_function(wrap_pyfunction!(sql::resolve_scenario_check_sql, module)?)?; - module.add_function(wrap_pyfunction!( - artifacts::compute_scenario_hash_prefix, - module - )?)?; - module.add_function(wrap_pyfunction!( - artifacts::build_scenario_hash_index, - module - )?)?; - module.add_function(wrap_pyfunction!( - artifacts::build_scenario_artifact_name, - module - )?)?; - module.add_function(wrap_pyfunction!( - artifacts::parse_scenario_artifact_physical_name, - module - )?)?; - module.add_function(wrap_pyfunction!( - artifacts::is_scenario_artifact_physical_name, - module - )?)?; - module.add_function(wrap_pyfunction!( - artifacts::build_scenario_relation_map, - module - )?)?; - module.add_function(wrap_pyfunction!(graph::plan_scenario_graph, module)?)?; - module.add_function(wrap_pyfunction!( - relations::build_scenario_relation_plan_of, - module - )?)?; - module.add_function(wrap_pyfunction!(cli::build_cli_scenario_plan, module)?)?; + module.add_function(build_scenario_execution_plan_function(module)?)?; + module.add_function(build_scenario_fixture_plans_function(module)?)?; + module.add_function(resolve_scenario_check_sql_function(module)?)?; + module.add_function(compute_scenario_hash_prefix_function(module)?)?; + module.add_function(build_scenario_hash_index_function(module)?)?; + module.add_function(build_scenario_artifact_name_function(module)?)?; + module.add_function(parse_scenario_artifact_physical_name_function(module)?)?; + module.add_function(scenario_artifact_physical_name_check_function(module)?)?; + module.add_function(build_scenario_relation_map_function(module)?)?; + module.add_function(plan_scenario_graph_function(module)?)?; + module.add_function(build_scenario_relation_plan_of_function(module)?)?; + module.add_function(build_cli_scenario_plan_function(module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs index 8176115560..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs @@ -1,11 +1 @@ -pub(crate) mod audit_entries; -pub(crate) mod audits; -pub(crate) mod cursor_inputs; -pub(crate) mod cursor_types; -pub(crate) mod lineage; -pub(crate) mod microbatch_limits; -pub(crate) mod microbatches; -pub(crate) mod model_entries; pub(crate) mod registration; -pub(crate) mod strategy; -pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index 5ae6a39806..4fed3db332 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -1,47 +1,36 @@ //! Register the planner strategy. +use pyo3::prelude::PyModuleMethods; use pyo3::prelude::{Bound, PyModule, PyResult}; -use pyo3::{prelude::PyModuleMethods, wrap_pyfunction}; -use crate::bindings::_helpers::planner_strategy::{audits, cursor_types, microbatches, strategy}; +use sqlbuild_python_planner::planner_strategy::main::audits::register_audits::register_audits; +use sqlbuild_python_planner::planner_strategy::main::cursor_types::register_cursor_types::register_cursor_types; +use sqlbuild_python_planner::planner_strategy::main::microbatches::register_microbatches::register_microbatches; +use sqlbuild_python_planner::planner_strategy::main::strategy::register_strategy::register_strategy; use sqlbuild_python_project::planner_strategy::main::register_fixture_completion::register_fixture_completion; use sqlbuild_python_project::planner_strategy::main::register_fixture_context::register_fixture_context; use sqlbuild_python_project::planner_strategy::main::register_fixture_reads::register_fixture_reads; +use sqlbuild_python_planner::planner_strategy::main::audit_entries::plan_audit_of_function::plan_audit_of_function; +use sqlbuild_python_planner::planner_strategy::main::cursor_inputs::cursor_input_columns_of_function::cursor_input_columns_of_function; +use sqlbuild_python_planner::planner_strategy::main::cursor_inputs::cursor_input_relations_of_function::cursor_input_relations_of_function; +use sqlbuild_python_planner::planner_strategy::main::cursor_inputs::runtime_cursor_producer_names_function::runtime_cursor_producer_names_function; +use sqlbuild_python_planner::planner_strategy::main::microbatch_limits::apply_configured_microbatch_limits_function::apply_configured_microbatch_limits_function; +use sqlbuild_python_planner::planner_strategy::main::microbatch_limits::apply_microbatch_limit_function::apply_microbatch_limit_function; +use sqlbuild_python_planner::planner_strategy::main::model_entries::resolve_microbatch_lookback_function::resolve_microbatch_lookback_function; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_strategy::audit_entries::plan_audit_of, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_strategy::cursor_inputs::cursor_input_columns_of, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_strategy::cursor_inputs::cursor_input_relations_of, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_strategy::cursor_inputs::runtime_cursor_producer_names, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_strategy::model_entries::resolve_microbatch_lookback, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_microbatch_limit, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_configured_microbatch_limits, - module - )?)?; - audits::register(module)?; - cursor_types::register(module)?; + module.add_function(plan_audit_of_function(module)?)?; + module.add_function(cursor_input_columns_of_function(module)?)?; + module.add_function(cursor_input_relations_of_function(module)?)?; + module.add_function(runtime_cursor_producer_names_function(module)?)?; + module.add_function(resolve_microbatch_lookback_function(module)?)?; + module.add_function(apply_microbatch_limit_function(module)?)?; + module.add_function(apply_configured_microbatch_limits_function(module)?)?; + register_audits(module)?; + register_cursor_types(module)?; register_fixture_completion(module)?; register_fixture_context(module)?; register_fixture_reads(module)?; - microbatches::register(module)?; - strategy::register(module) + register_microbatches(module)?; + register_strategy(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs index 69d9ef9fe4..da41dfcf50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs @@ -1,14 +1,3 @@ //! Native warehouse inspection for planning over Python adapter callbacks. -pub(crate) mod constants; -pub(crate) mod cursors; -pub(crate) mod fingerprint_cache; -pub(crate) mod models; pub(crate) mod registration; -pub(crate) mod semantic_sources; -pub(crate) mod snapshot; -pub(crate) mod source_columns; -pub(crate) mod source_deferral; -pub(crate) mod source_freshness; -pub(crate) mod source_tables; -pub(crate) mod watermarks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/models.rs deleted file mode 100644 index 6c19c1fcea..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/models.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Warehouse planning state shared across planner runs. - -use std::collections::BTreeSet; - -use pyo3::pyclass; - -/// Migration fingerprints computed since the last reset, for work-count tests. -#[pyclass] -#[derive(Debug, Default)] -pub(crate) struct FingerprintWork { - pub(crate) computations: u64, - pub(crate) distinct: BTreeSet, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs index 5b7ca3db79..f2cf7b6049 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs @@ -1,44 +1,27 @@ //! Register native warehouse inspection for planning. use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::wrap_pyfunction; +use sqlbuild_python_planner::planner_warehouse::main::inspection::gather_warehouse_snapshot_function::gather_warehouse_snapshot_function; +use sqlbuild_python_planner::planner_warehouse::main::inspection::migration_fingerprint_work_info_function::migration_fingerprint_work_info_function; +use sqlbuild_python_planner::planner_warehouse::main::inspection::planner_metadata_name_filter_function::planner_metadata_name_filter_function; +use sqlbuild_python_planner::planner_warehouse::main::inspection::watermark_type_is_compatible_function::watermark_type_is_compatible_function; +use sqlbuild_python_planner::planner_warehouse::main::sources::build_planner_source_freshness_result_of_function::build_planner_source_freshness_result_of_function; +use sqlbuild_python_planner::planner_warehouse::main::sources::check_selected_source_tables_exist_binding_function::check_selected_source_tables_exist_binding_function; +use sqlbuild_python_planner::planner_warehouse::main::sources::gather_source_columns_function::gather_source_columns_function; +use sqlbuild_python_planner::planner_warehouse::main::sources::list_source_candidates_function::list_source_candidates_function; +use sqlbuild_python_planner::planner_warehouse::main::sources::semantic_source_columns_function::semantic_source_columns_function; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!( - super::fingerprint_cache::migration_fingerprint_work_info, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::snapshot::gather_warehouse_snapshot, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::watermarks::watermark_type_is_compatible, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::source_columns::gather_source_columns, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::snapshot::planner_metadata_name_filter, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::semantic_sources::list_source_candidates, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::semantic_sources::semantic_source_columns, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::source_tables::check_selected_source_tables_exist_binding, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::source_freshness::build_planner_source_freshness_result_of, - module - )?) + module + .add_class::( + )?; + module.add_function(migration_fingerprint_work_info_function(module)?)?; + module.add_function(gather_warehouse_snapshot_function(module)?)?; + module.add_function(watermark_type_is_compatible_function(module)?)?; + module.add_function(gather_source_columns_function(module)?)?; + module.add_function(planner_metadata_name_filter_function(module)?)?; + module.add_function(list_source_candidates_function(module)?)?; + module.add_function(semantic_source_columns_function(module)?)?; + module.add_function(check_selected_source_tables_exist_binding_function(module)?)?; + module.add_function(build_planner_source_freshness_result_of_function(module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index 42514a58ab..73a8bcc57d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -1,6 +1,5 @@ //! Native behaviour of the hosted shapes: computed attributes, methods, ordering and validation. -use crate::bindings::_helpers::freshness::observation::normalize_observation; use crate::bindings::_helpers::record_behaviours::auditing::{ finish_measurement_contract, finish_threshold_bound, finish_thresholds, }; @@ -71,13 +70,13 @@ use crate::bindings::_helpers::record_behaviours::values::{ }; use crate::bindings::_helpers::{ analysis, analysis_session, build_progress, compile_command, compute_logs, dbt_events, - dbt_interop, event_exporting, execution_history, planner_entries, planner_migrations, - planner_phases, project_assembly, result_output, rules, + dbt_interop, event_exporting, execution_history, project_assembly, result_output, rules, }; use sqlbuild_python_base::records::models::{GetterEntry, RecordBehaviours}; use sqlbuild_python_base::records::types::ClassMethod; use sqlbuild_python_base::records::types::ClassProperty; use sqlbuild_python_base::records::types::RecordFinish; +use sqlbuild_python_planner::freshness::main::normalize_observation::normalize_observation; use sqlbuild_python_project::lineage::main::facts_eq::facts_eq; use sqlbuild_python_project::lineage::main::facts_getitem::facts_getitem; use sqlbuild_python_project::lineage::main::facts_iter::facts_iter; @@ -372,9 +371,9 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ execution_history::getters::GETTERS, sqlbuild_python_project::graph::constants::GETTERS, sqlbuild_python_front::model_config::constants::GETTERS, - planner_entries::getters::GETTERS, - planner_migrations::getters::GETTERS, - planner_phases::getters::GETTERS, + sqlbuild_python_planner::planner_entries::constants::GETTERS, + sqlbuild_python_planner::planner_migrations::constants::GETTERS, + sqlbuild_python_planner::planner_phases::constants::GETTERS, sqlbuild_python_base::presentation_text::constants::GETTERS, project_assembly::getters::GETTERS, result_output::getters::GETTERS, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/freshness.rs index 14b1ae4809..13b1403a95 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/freshness.rs @@ -3,10 +3,10 @@ use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::types::PyDict; -use crate::bindings::_helpers::freshness::observation::require_aware_utc_datetime; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_planner::freshness::main::require_aware_utc_datetime::require_aware_utc_datetime; /// `SourceFreshnessRecord.observed_at` position. const OBSERVED_AT_FIELD: usize = 9; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs index f11dfcc336..f9b8acfd88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs @@ -8,12 +8,15 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::plan_output::safety::{future_cursor_safety, maximum_start_safety}; -use crate::bindings::_helpers::plan_output::values::{any, dict, items}; use crate::bindings::_helpers::result_output::projection::{Projection, rounded}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_planner::plan_output::main::any::any; +use sqlbuild_python_planner::plan_output::main::dict::dict; +use sqlbuild_python_planner::plan_output::main::future_cursor_safety::future_cursor_safety; +use sqlbuild_python_planner::plan_output::main::items::items; +use sqlbuild_python_planner::plan_output::main::maximum_start_safety::maximum_start_safety; /// A dictionary of the entries whose value is not `None`, in order. pub(crate) fn without_none<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs index 3b13e982ad..20dc6a7cc0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs @@ -9,11 +9,14 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::plan_output::values::{any, dict, items, text}; use crate::bindings::_helpers::result_output::assets::{or_none, without_none}; use crate::bindings::_helpers::result_output::projection::{Projection, ResultIdentity, rounded}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::scopes::main::reports::json_to_python::json_to_python; +use sqlbuild_python_planner::plan_output::main::any::any; +use sqlbuild_python_planner::plan_output::main::dict::dict; +use sqlbuild_python_planner::plan_output::main::items::items; +use sqlbuild_python_planner::plan_output::main::text::text; const DECIMAL_KIND: &str = "decimal"; const SCALAR_KINDS: [&str; 5] = ["string", "integer", "boolean", "float", "null"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs index ff60788c89..3ff857802d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs @@ -10,7 +10,6 @@ use sqlbuild_host::models::{ SqlTestExecutionResult, SqlTestOutcome, }; -use crate::bindings::_helpers::plan_output::values::{any, dict, items, text}; use crate::bindings::_helpers::result_output::aggregation::RetainedBuildResults; use crate::bindings::_helpers::result_output::assets::{ clone_asset, function_assets, load_assets, model_assets, python_node_assets, seed_assets, @@ -25,6 +24,10 @@ use sqlbuild_python_base::compile_outputs::main::indented_json::indented_json; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_planner::plan_output::main::any::any; +use sqlbuild_python_planner::plan_output::main::dict::dict; +use sqlbuild_python_planner::plan_output::main::items::items; +use sqlbuild_python_planner::plan_output::main::text::text; const JSON_VERSION: i64 = 1; const FAILED: &str = "failed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs index f820e6902f..8c098250dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs @@ -14,8 +14,6 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::plan_output::safety::{future_cursor_safety, maximum_start_safety}; -use crate::bindings::_helpers::plan_output::values::any; use crate::bindings::_helpers::result_output::assets::microbatch_summary; use crate::bindings::_helpers::result_output::checks::{ measured_value, measurement_thresholds, python_check_status, @@ -32,6 +30,9 @@ use sqlbuild_python_base::result_output::main::required_json::required_json; use sqlbuild_python_base::result_output::main::text::text; use sqlbuild_python_base::result_output::main::text_or_empty::text_or_empty; use sqlbuild_python_base::result_output::types::Checked; +use sqlbuild_python_planner::plan_output::main::any::any; +use sqlbuild_python_planner::plan_output::main::future_cursor_safety::future_cursor_safety; +use sqlbuild_python_planner::plan_output::main::maximum_start_safety::maximum_start_safety; const FAILED_EVENT: &str = "resource_attempt_failed"; const SKIPPED_EVENT: &str = "resource_attempt_skipped"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs index a35bae9289..d9228df633 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs @@ -11,11 +11,11 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::plan_output::values::text; use crate::bindings::_helpers::result_output::resource_ids::{audit_identity, sql_test_identity}; use crate::bindings::_helpers::result_output::terminal_index::{TerminalEventIndex, current_index}; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_planner::plan_output::main::text::text; /// The executor result types that result output distinguishes. #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 95b3ab4c7e..28c9cb55df 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -23,8 +23,6 @@ use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pi use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::planner_entries::command_plans::build_seed_command_plan; -use crate::bindings::_helpers::planner_entries::command_plans::read_selected_seed_fingerprints; use crate::bindings::_helpers::result_output::documents::format_seed_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; @@ -38,6 +36,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_planner::planner_entries::main::plans::build_seed_command_plan::build_seed_command_plan; +use sqlbuild_python_planner::planner_entries::main::plans::read_selected_seed_fingerprints::read_selected_seed_fingerprints; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; use sqlbuild_python_project::project_snapshot::types::VerifiedRun; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 91a8fbf00a..55b78be584 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -43,7 +43,6 @@ use crate::bindings::_helpers::compile_driver::static_context::compile_static_co use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runtime_target; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::native_host::statement_report::StatementReport; -use crate::bindings::_helpers::planner_entries::command_plans::build_test_command_plan; use crate::bindings::_helpers::result_output::documents::test_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; @@ -72,6 +71,7 @@ use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_planner::planner_entries::main::plans::build_test_command_plan::build_test_command_plan; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; use sqlbuild_python_project::project_snapshot::types::VerifiedRun; diff --git a/fensu.toml b/fensu.toml index c4f6c000b9..667fedb53c 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1670,12 +1670,12 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 147789 } +thresholds = { max_crate_code_lines = 115238 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/src/bindings/_helpers"] -thresholds = { max_helpers_container_buckets = 98 } +thresholds = { max_helpers_container_buckets = 96 } reason = "The bindings helpers container is above the bucket limit; this caps it at its current bucket count until cohesive groups move into their own modules or crates. Lower the cap as buckets move out." [targets.rust.rule_options.XSB076] @@ -1702,7 +1702,7 @@ allowed_paths = [ "crates/sqlbuild-python-front/src/native_host/main/append_observations.rs=Writes observations after a build.", "crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs=Registers the observation state bindings.", "crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs=Registers the observation state bindings.", - "crates/sqlbuild-python/src/bindings/_helpers/observations=Writes observations after a build.", + "crates/sqlbuild-python-planner/src/observations=Writes observations after a build.", "crates/sqlbuild-python-front/src/native_host/main/mod.rs=Declares the observation write and registration entries.", ] @@ -1710,7 +1710,7 @@ allowed_paths = [ allowed_paths = [ "crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs=A location's requested plan names the synced copy for its sqb plan child.", "crates/sqlbuild-core/src/plan_display/constants.rs=Names the synced-copy environment variable.", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs=Preview branch: a refused freshness observation leaves the source unknown.", + "crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs=Preview branch: a refused freshness observation leaves the source unknown.", "crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs=Registers the synced copy's refusal error with Python.", "crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs=Defines the synced-copy adapter.", "crates/sqlbuild-python-base/src/warehouse_access/main/register_synced_copy.rs=Registers the synced copy's refusal error with Python.", @@ -1718,12 +1718,12 @@ allowed_paths = [ "crates/sqlbuild-python-base/src/warehouse_access/main/is_warehouse_access_refused.rs=Tells a plan preview's refused warehouse access apart.", "crates/sqlbuild-python-base/src/warehouse_access/models.rs=Defines the synced-copy adapter the plan preview reads through.", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=Only sqb plan activates the synced copy and reports what it could not know.", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=Preview branch: target MAX from the copy, watermark inputs read when the run starts.", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=Preview branch: table types and retention stay unknown.", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs=Preview branch: cursor intrinsics bind to bounds the run resolves.", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs=Preview branch: expression columns from the copy instead of DESCRIBE.", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs=Preview branch: the copy has no source freshness state to read.", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs=Preview branch: sources outside the copy are not probed.", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs=Preview branch: target MAX from the copy, watermark inputs read when the run starts.", + "crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs=Preview branch: table types and retention stay unknown.", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs=Preview branch: cursor intrinsics bind to bounds the run resolves.", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs=Preview branch: expression columns from the copy instead of DESCRIBE.", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs=Preview branch: the copy has no source freshness state to read.", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs=Preview branch: sources outside the copy are not probed.", "crates/sqlbuild-python-front/src/native_host/models.rs=Preview branch: --changed reads consumption history from the copy instead of the warehouse table.", ] @@ -2022,6 +2022,7 @@ crate_names = [ "sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", + "sqlbuild-python-planner", "sqlbuild-python", "sqlbuild-recall", "sqlbuild-refactor", @@ -2047,7 +2048,7 @@ paths = ["src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py"] reason = "SQL Server overrides each state table's create SQL to add that table's own key index; the keyed DDL lives in _keyed_state_table_sql." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs", "crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs"] +paths = ["crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs", "crates/sqlbuild-python-base/src/warehouse_access/_helpers/synced_copy.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs"] reason = "Each turns its own record type into its own Python objects; only the loop that collects them is alike." [[dupes.allowlist]] @@ -2152,6 +2153,11 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs=3", "crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs=1", "crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs=5", + "crates/sqlbuild-python-front/src/cost_context/_helpers/context.rs=3", + "crates/sqlbuild-python-front/src/cost_context/main/instant_micros.rs=6", + "crates/sqlbuild-python-front/src/cost_context/main/native_decimal.rs=1", + "crates/sqlbuild-python-front/src/cost_context/main/python_decimal.rs=2", + "crates/sqlbuild-python-front/src/cost_context/models.rs=2", "crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs=0", "crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs=1", "crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs=0", @@ -2265,6 +2271,115 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs=1", "crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs=1", "crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs=1", + "crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs=6", + "crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs=3", + "crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs=1", + "crates/sqlbuild-python-planner/src/freshness/_helpers/publication.rs=6", + "crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs=1", + "crates/sqlbuild-python-planner/src/freshness/_helpers/support.rs=0", + "crates/sqlbuild-python-planner/src/freshness/main/append_freshness_records.rs=3", + "crates/sqlbuild-python-planner/src/freshness/main/presumed_utc_datetime.rs=2", + "crates/sqlbuild-python-planner/src/freshness/main/row_record.rs=2", + "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs=4", + "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/payloads.rs=4", + "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs=4", + "crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs=0", + "crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs=1", + "crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs=5", + "crates/sqlbuild-python-planner/src/microbatches/main/timestamp.rs=1", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs=5", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs=9", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs=1", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs=7", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs=5", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/python_identity.rs=2", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs=1", + "crates/sqlbuild-python-planner/src/plan_output/main/resolve_effective_microbatch_batch_size.rs=1", + "crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs=5", + "crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs=2", + "crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs=4", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs=2", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs=1", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/values.rs=1", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs=5", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs=3", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs=1", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs=2", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs=0", + "crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_test_planning.rs=2", + "crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_command_scope.rs=1", + "crates/sqlbuild-python-planner/src/planner_identity/_helpers/identities.rs=0", + "crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs=6", + "crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs=1", + "crates/sqlbuild-python-planner/src/planner_identity/main/hook_version_hashes.rs=1", + "crates/sqlbuild-python-planner/src/planner_identity/main/merge_recomputed_model_changes.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs=3", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs=3", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs=8", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs=6", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_models.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs=9", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs=5", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs=4", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs=13", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs=4", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs=7", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs=7", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs=6", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/query_shapes.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/models.rs=1", + "crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs=2", + "crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs=1", + "crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs=3", + "crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs=2", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_bounds.rs=2", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_model.rs=3", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_overrides.rs=2", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_policies.rs=2", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs=5", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_safety.rs=2", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_windows.rs=1", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs=1", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs=4", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs=1", + "crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs=3", + "crates/sqlbuild-python-planner/src/planner_scenario/_helpers/cli.rs=1", + "crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs=1", + "crates/sqlbuild-python-planner/src/planner_scenario/_helpers/execution.rs=1", + "crates/sqlbuild-python-planner/src/planner_scenario/_helpers/sql.rs=1", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs=4", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs=8", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/cursor_types.rs=2", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs=1", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs=3", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs=0", + "crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/audit_is_selected.rs=2", + "crates/sqlbuild-python-planner/src/planner_strategy/main/values/input_error.rs=1", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs=14", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs=6", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs=3", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/watermarks.rs=2", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/semantic_sources.rs=3", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs=1", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_deferral.rs=1", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs=5", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs=5", + "crates/sqlbuild-python-planner/src/planner_warehouse/constants.rs=1", + "crates/sqlbuild-python-planner/src/planner_warehouse/main/sources/source_freshness_unknown_warnings.rs=1", + "crates/sqlbuild-python-planner/src/planner_warehouse/models.rs=8", "crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs=0", "crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs=1", "crates/sqlbuild-python-project/src/analysis/models.rs=1", @@ -2435,11 +2550,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=23", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs=7", @@ -2495,12 +2608,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs=17", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs=3", @@ -2535,9 +2642,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_plan.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs=12", @@ -2551,101 +2655,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=31", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs=3", @@ -2769,6 +2784,7 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/authored_values/models.rs=6", "crates/sqlbuild-python-front/src/compile_driver/models.rs=3", "crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs=1", + "crates/sqlbuild-python-front/src/cost_context/models.rs=2", "crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs=1", "crates/sqlbuild-python-front/src/cursor/models.rs=4", "crates/sqlbuild-python-front/src/discovery/_helpers/failures/faults.rs=2", @@ -2805,6 +2821,30 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/scopes/models.rs=11", "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs=1", "crates/sqlbuild-python-front/src/state_rows/models.rs=3", + "crates/sqlbuild-python-planner/src/freshness/models.rs=8", + "crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs=1", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs=2", + "crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs=2", + "crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs=1", + "crates/sqlbuild-python-planner/src/planner_changes/models.rs=3", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs=6", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs=9", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs=2", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs=2", + "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs=4", + "crates/sqlbuild-python-planner/src/planner_entries/models.rs=56", + "crates/sqlbuild-python-planner/src/planner_identity/_helpers/staleness.rs=4", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/known_input_columns.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs=2", + "crates/sqlbuild-python-planner/src/planner_migrations/models.rs=77", + "crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs=4", + "crates/sqlbuild-python-planner/src/planner_phases/models.rs=38", + "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs=3", + "crates/sqlbuild-python-planner/src/planner_resolution/models.rs=20", + "crates/sqlbuild-python-planner/src/planner_scenario/models.rs=9", + "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/lineage.rs=1", + "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/cursors.rs=19", + "crates/sqlbuild-python-planner/src/planner_warehouse/models.rs=11", "crates/sqlbuild-python-project/src/analysis/models.rs=9", "crates/sqlbuild-python-project/src/analysis_session/models.rs=1", "crates/sqlbuild-python-project/src/compiled_project/models.rs=36", @@ -2817,6 +2857,7 @@ pending_budgets = [ "crates/sqlbuild-python-project/src/graph/models.rs=10", "crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs=2", "crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs=1", + "crates/sqlbuild-python-project/src/planner_scope/models.rs=3", "crates/sqlbuild-python-project/src/planner_selection/models.rs=1", "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs=6", "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs=3", @@ -2876,7 +2917,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs=1", @@ -2896,9 +2936,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs=22", @@ -2925,35 +2962,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=56", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs=66", - "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=41", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=19", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=3", @@ -3030,8 +3042,6 @@ module_layers = [ "cost_command=commands", "cost_report=output", "cost_telemetry=output", - "cost_telemetry/context=front", - "cost_telemetry/values=front", "cursor=registry", "dag_artifact=compile", "dagster_invocation=commands", @@ -3056,14 +3066,14 @@ module_layers = [ "init_command=commands", "input_loops=compile", "janitor=commands", - "lifecycle_events=plan", + "lifecycle_events=registry", "lineage=compile", "lineage/lineage_command=commands", "load=commands", "location_command=commands", "macro_bridge=registry", "manifest=compile", - "microbatches=plan", + "microbatches=registry", "model_config=registry", "native_host=front", "native_host/orchestration_sensors=commands", @@ -3072,21 +3082,19 @@ module_layers = [ "observations=plan", "output_capture=output", "plan_command=commands", - "plan_output=plan", - "planner_changes=plan", - "planner_entries=plan", - "planner_identity=plan", - "planner_migrations=plan", - "planner_phases=plan", - "planner_phases/models=project", - "planner_phases/scopes=project", + "plan_output=registry", + "planner_changes=registry", + "planner_entries=registry", + "planner_identity=registry", + "planner_migrations=registry", + "planner_phases=registry", "planner_registration=registry", - "planner_resolution=plan", - "planner_scenario=plan", + "planner_resolution=registry", + "planner_scenario=registry", "planner_scope=registry", "planner_selection=registry", - "planner_strategy=plan", - "planner_warehouse=plan", + "planner_strategy=registry", + "planner_warehouse=registry", "presentation_text=registry", "project_assembly=compile", "query_command=commands", @@ -3156,7 +3164,7 @@ paths = ["crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs", "c reason = "Field lookups on unrelated value types in different crates: project configuration takes the first matching field of a built object, while rule headers take the last entry of an authored map so a repeated key overrides earlier ones." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs", "crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs"] +paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs", "crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs"] reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." [[dupes.allowlist]] @@ -3168,7 +3176,7 @@ paths = ["crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registra reason = "Each bindings module registers its own functions and classes with the native module." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/diagnostics/_helpers/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/getters.rs"] +paths = ["crates/sqlbuild-python-base/src/diagnostics/_helpers/getters.rs", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs"] reason = "Direct `#[getter]` readers of each module's own pyclass; each matches its own class and field names." [[dupes.allowlist]] @@ -3192,11 +3200,11 @@ paths = ["crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs", "crates reason = "Each maps a Python `str` into its own JSON value type; the UTF-16 conversion itself is shared through `utf16_units`." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/warehouse_access/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs"] +paths = ["crates/sqlbuild-python-base/src/warehouse_access/models.rs", "crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs"] reason = "Each turns its own record type into its own Python objects; only the loop that collects them is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs"] +paths = ["crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs", "crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs"] reason = "The capture encoder sorts by canonical capture key and freshness planning by resource identity; only the decorate-sort-undecorate shape is alike." [[dupes.allowlist]] @@ -3228,7 +3236,7 @@ paths = ["crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs", reason = "Each reads its own Python record's fields into its own native record." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs", "crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs"] +paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs", "crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs"] reason = "Each bindings module registers its own functions and classes with the native module." [[dupes.allowlist]] @@ -3260,7 +3268,7 @@ paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/model_input reason = "Each borrows its own record from the compiled project owner by index; only the borrow-and-read shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs"] +paths = ["crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/python_identity.rs"] reason = "Each reads an optional text value from its own Python mapping; consolidating these readers across the binding crates is a separate change." [[dupes.allowlist]] @@ -3296,7 +3304,7 @@ paths = ["crates/sqlbuild-python-project/src/project_snapshot/models.rs"] reason = "Each worker report forwards one distinct check event with its own fields." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs"] +paths = ["crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs"] reason = "One returns the first model destination's database, the other collects every model destination's schema; only the loop over compiled models is alike." [[dupes.allowlist]] @@ -3332,19 +3340,19 @@ paths = ["crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs" reason = "Each public entry forwards to its own helper with the same arguments." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs"] -reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs", "crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs"] +paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs"] reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs", "crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs"] +paths = ["crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs", "crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/envelopes.rs"] reason = "Each bindings module registers its own functions and classes with the native module." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs"] +paths = ["crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/python_identity.rs"] reason = "Each reads an optional text value from its own Python mapping; consolidating these readers across the binding crates is a separate change." [[dupes.allowlist]] @@ -3388,7 +3396,7 @@ paths = ["crates/sqlbuild-python-project/src/resource_views/main/audit_reads/alw reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/oracles.rs", "crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs", "crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs", "crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs", "crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/oracles.rs", "crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs", "crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs", "crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs", "crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs", "crates/sqlbuild-python-planner/src/freshness/_helpers/state_io.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3396,15 +3404,15 @@ paths = ["crates/sqlbuild-python-front/src/attachments/main/register_cursor_intr reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs"] +paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs"] reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs"] +paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs"] +paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3412,11 +3420,11 @@ paths = ["crates/sqlbuild-python-project/src/resource_views/main/resource_reads/ reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs", "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs"] +paths = ["crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs", "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs"] reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs"] +paths = ["crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs"] reason = "Each reads the installed package version for its own output with its own fallback; only the metadata lookup is alike." [[dupes.allowlist]] @@ -3436,7 +3444,7 @@ paths = ["crates/sqlbuild-python-project/src/compiled_views/main/project/audits. reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs"] +paths = ["crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs"] reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." [[dupes.allowlist]] @@ -3472,7 +3480,7 @@ paths = ["crates/sqlbuild-python-front/src/model_config/_helpers/config/config_v reason = "Each checks one config key as its own value type; only the node lookup and error mapping are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/python_calls.rs", "crates/sqlbuild-python-project/src/planner_phases/main/replace.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/python_calls.rs", "crates/sqlbuild-python-project/src/planner_phases/main/replace.rs", "crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/values.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_safety.rs"] reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." [[dupes.allowlist]] @@ -3560,7 +3568,7 @@ paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migrat reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/planner_scope/main/edges.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs"] +paths = ["crates/sqlbuild-python-project/src/planner_scope/main/edges.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/_helpers/payloads.rs"] reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] @@ -3580,7 +3588,7 @@ paths = ["crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.r reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs"] +paths = ["crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs", "crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs"] reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] @@ -3608,7 +3616,7 @@ paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshn reason = "Each builds its own error class's exception from a message; sharing one exception constructor across the binding crates is a separate change." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs"] +paths = ["crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_columns.rs"] reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] @@ -3619,6 +3627,102 @@ reason = "Each converts its own input into its own native or Python value; only paths = ["crates/sqlbuild-python-front/src/render/models.rs", "crates/sqlbuild-python-project/src/analysis_session/models.rs", "crates/sqlbuild-python-project/src/compiled_project/models.rs"] reason = "Each positional record converts its own fields; models.rs holds only records, so these are the conversions positional_record! writes, written out once per record." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/oracles.rs", "crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs", "crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs", "crates/sqlbuild-python-planner/src/freshness/main/register_state_io.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/main/register_validation.rs", "crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs", "crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs", "crates/sqlbuild-python-planner/src/plan_output/main/dict.rs", "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs"] +reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs", "crates/sqlbuild-python-planner/src/lifecycle_events/main/register_envelopes.rs", "crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs"] +reason = "Each bindings module registers its own functions and classes with the native module." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_identity/main/or_empty.rs", "crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/getters.rs", "crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/getters.rs"] +reason = "Each matches its own native class's attribute names to its own fields; only the cast-and-dispatch shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/getters.rs"] +reason = "Each matches its own native class's attribute names to its own fields; only the cast-and-dispatch shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/microbatches/main/register_statements.rs", "crates/sqlbuild-python-planner/src/planner_entries/main/plans/register_command_plans.rs", "crates/sqlbuild-python-planner/src/planner_identity/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_windows.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/getters.rs"] +reason = "Each matches its own native class's attribute names to its own fields; only the cast-and-dispatch shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs"] +reason = "Each writes the entries of its own plan section with its own keys; only mapping over the entries is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/contexts.rs", "crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs"] +reason = "runtime_field and warehouse_field are class_field fixed to one planner class; moving their call sites to class_field is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/relations.rs", "crates/sqlbuild-python-planner/src/planner_entries/main/plans/resolve_static_relation_context.rs"] +reason = "The full relations context reads the connection and known source columns from keywords and the static one fills them with empty values; only the keyword parsing is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/config_str.rs"] +reason = "Each planner area reads one model config value with its own copy of the reader; sharing one reader across the planner areas is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs"] +reason = "Each maps its own enum's variants to its own labels; only the match shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mock_reads.rs", "crates/sqlbuild-analysis/src/required_columns/_helpers/relations.rs", "crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs"] +reason = "Each replaces or appends a value in its own ordered entry list; the types are separate and live in separate crates." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/sources.rs"] +reason = "Each Python entry reads its own keyword arguments and calls its own resolver; only the keyword parsing is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs"] +reason = "Each writes its own record's fields as a JSON dict; only building the dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/microbatches/_helpers/statements.rs"] +reason = "One renders the insert statement for the events and the other the lookup of their IDs; only the events-table set-up is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_migrations/models.rs"] +reason = "Each looks up its own map by relation key with its own default; only the lookup is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs", "crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/fingerprint_cache.rs"] +reason = "Each reads and resets its own work counters; only the take-and-reset shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs"] +reason = "One observes one configured source and the other a batch of adapter sources; only building the observer is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs"] +reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." + [[dupes.allowlist]] paths = ["crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs", "crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs"] reason = "Each is one rule family's entry point: return early when no rule is selected, then run its own layout, module and import checks. Parsing, project modules and import resolution share files_naming, project_modules and rejected_import_faults." diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 3c7b02624c..695d7a15a3 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -288,6 +288,7 @@ "crates/sqlbuild-python-base/src/", "crates/sqlbuild-python-front/src/", "crates/sqlbuild-python-project/src/", + "crates/sqlbuild-python-planner/src/", ) NATIVE_CROSSING_PATTERNS: tuple[str, ...] = ( r"\.getattr\(", diff --git a/scripts/native_module_paths/constants.py b/scripts/native_module_paths/constants.py index 904f3f4ce6..e426c4f2b7 100644 --- a/scripts/native_module_paths/constants.py +++ b/scripts/native_module_paths/constants.py @@ -6,6 +6,7 @@ "crates/sqlbuild-python-base/src", "crates/sqlbuild-python-front/src", "crates/sqlbuild-python-project/src", + "crates/sqlbuild-python-planner/src", "crates/sqlbuild-python/src", "crates/sqlbuild-cli/src", "crates/sqlbuild-host/src", diff --git a/tests/native_field_fallbacks.txt b/tests/native_field_fallbacks.txt index 26f2188a9b..08510b752e 100644 --- a/tests/native_field_fallbacks.txt +++ b/tests/native_field_fallbacks.txt @@ -188,9 +188,9 @@ crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs Dec crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs ScopeKind.value crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs EventExporterCommandScope.cached_extensions crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs TransientStatusReporter.complete -crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs SourceFreshnessAgeStatus.value -crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs SourceFreshnessStrategy.value -crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs SourceFreshnessValueKind.value +crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs SourceFreshnessAgeStatus.value +crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs SourceFreshnessStrategy.value +crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs SourceFreshnessValueKind.value crates/sqlbuild-python-base/src/frontier/models.rs BackfillAction.name crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs BindingCatalog.analysis_shapes crates/sqlbuild-python-base/src/frontier/models.rs BindingCatalog.analysis_shapes @@ -261,117 +261,124 @@ crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs ConnectionProg crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs ConnectionProgressReporter.on_connection_error crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs ConnectionProgressReporter.on_connection_start crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs PlanningProgressReporter.on_progress -crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs MigrationStateInspection.inspect_migration_events -crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs PlannerRuntime.project -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs MaterializationType.value -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs PlannerRelationsContext.function_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs PlannerRelationsContext.model_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs PlannerRelationsContext.python_source_read_map -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs PlannerRelationsContext.seed_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs PlannerRelationsContext.source_map -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs PlannerRelationsContext.source_read_map -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.function_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.model_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.python_source_read_map -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.seed_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.source_map -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.source_read_map -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.source_warehouse_columns -crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs PlannerRelationsContext.star_exclude_keyword -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs ColumnMigrationDecision.blocks_build -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs ColumnMigrationDecision.renames -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs ColumnMigrationEvent.renamed -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs PlannerRuntime.adapter -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs PlannerRuntime.project -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs ColumnMigrationDecision.renames -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs MigrationStateInspection.inspect_column_events -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs MigrationStateInspection.model_fingerprint -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs PlannerRuntime.project -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs MigrationStateInspection.inspect_relations -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs MigrationStateInspection.orphan_fingerprints -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs MigrationStateInspection.inspect_schemas -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs MigrationStateInspection.named_fingerprint -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs PlannerRuntime.adapter -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs MigrationFingerprintCache.fingerprint -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs MigrationFingerprintCache.fingerprint -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs MigrationDecision.moves_data -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs MigrationStateInspection.model_fingerprint -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs PlannerRuntime.project -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs MigrationDecision.moves_data -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs MigrationStateInspection.inspect_relations -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs MigrationStateInspection.inspect_schemas -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs MigrationDecision.moves_data -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs OldNameViewHistory.status -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs PlannerRuntime.adapter -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs MigrationStateInspection.inspect_old_name_views -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs PlannerRuntime.project -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs PlannerRuntime.project -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs MigrationStateInspection.named_fingerprint -crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs PlannerRuntime.adapter -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs PlannerRelationsContext.source_warehouse_columns -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs PlannerWarehouseState.inspection_relations -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs PlannerRuntime.adapter -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs SourceFreshnessUnknownReason.value -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs PlannerRuntime.adapter -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs PlannerRuntime.project -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs PlannerWarehouseState.snapshot -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs RetentionDecreasePolicy.value -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs TableTypeDowngradePolicy.value -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs TableTypeSource.value -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs TableType.value -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs TimeTravelRetentionSource.value -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs MigrationDecision.moves_data -crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs PlannerRelationsContext.source_warehouse_columns -crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs FutureCursorAction.value -crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs CursorOverrides.typed_as -crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs DbtCompileReferenceResolver.resolve_reference -crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs ScenarioArtifactKind.value -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs PlannerRelationsContext.model_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs PlannerRelationsContext.seed_locations -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs PlannerRelationsContext.source_read_map +crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs MigrationStateInspection.inspect_migration_events +crates/sqlbuild-python-planner/src/planner_changes/_helpers/renames.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_audit_command_plan.rs MaterializationType.value +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs PlannerRelationsContext.function_locations +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs PlannerRelationsContext.model_locations +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs PlannerRelationsContext.python_source_read_map +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs PlannerRelationsContext.seed_locations +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs PlannerRelationsContext.source_map +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/command_plans.rs PlannerRelationsContext.source_read_map +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.function_locations +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.model_locations +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.python_source_read_map +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.seed_locations +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.source_map +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.source_read_map +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.source_warehouse_columns +crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs PlannerRelationsContext.star_exclude_keyword +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs ColumnMigrationDecision.blocks_build +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs ColumnMigrationDecision.renames +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs ColumnMigrationEvent.renamed +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs PlannerRuntime.adapter +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs ColumnMigrationDecision.renames +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs MigrationStateInspection.inspect_column_events +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs MigrationStateInspection.model_fingerprint +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs MigrationStateInspection.inspect_relations +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs MigrationStateInspection.orphan_fingerprints +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs MigrationStateInspection.inspect_schemas +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs MigrationStateInspection.named_fingerprint +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs PlannerRuntime.adapter +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/matching.rs MigrationFingerprintCache.fingerprint +crates/sqlbuild-python-planner/src/planner_migrations/main/with_migration_fingerprints.rs MigrationFingerprintCache.fingerprint +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs MigrationDecision.moves_data +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs MigrationStateInspection.model_fingerprint +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs MigrationDecision.moves_data +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs MigrationStateInspection.inspect_relations +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs MigrationStateInspection.inspect_schemas +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs MigrationDecision.moves_data +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs OldNameViewHistory.status +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs PlannerRuntime.adapter +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs MigrationStateInspection.inspect_old_name_views +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs MigrationStateInspection.named_fingerprint +crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs PlannerRuntime.adapter +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs PlannerRelationsContext.source_warehouse_columns +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs PlannerWarehouseState.inspection_relations +crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs PlannerRuntime.adapter +crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/output_assembly.rs SourceFreshnessUnknownReason.value +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs PlannerRuntime.adapter +crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/runtime.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/identities.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effective_target_config.rs PlannerRuntime.project +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs PlannerWarehouseState.snapshot +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs RetentionDecreasePolicy.value +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs TableTypeDowngradePolicy.value +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs TableTypeSource.value +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs TableType.value +crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/retention.rs TimeTravelRetentionSource.value +crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs MigrationDecision.moves_data +crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs PlannerRelationsContext.source_warehouse_columns +crates/sqlbuild-python-planner/src/planner_resolution/_helpers/safety/maximum_safety.rs FutureCursorAction.value +crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs CursorOverrides.typed_as +crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/values.rs DbtCompileReferenceResolver.resolve_reference +crates/sqlbuild-python-planner/src/planner_scenario/_helpers/artifacts.rs ScenarioArtifactKind.value +crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs PlannerRelationsContext.model_locations +crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs PlannerRelationsContext.model_locations +crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs PlannerRelationsContext.seed_locations +crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs PlannerRelationsContext.seed_locations +crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs PlannerRelationsContext.source_read_map crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_values.rs CompiledResourceType.value -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs PlanEntryBuildInputs.max_microbatches -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs PlanEntryBuildInputs.max_microbatches_is_override -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs PlanEntryBuildInputs.microbatch_limit_action -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs TableType.value -crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs PlannerRelationsContext.source_read_map -crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs PlannerRelationsContext.listed_source_names -crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs PlannerRelationsContext.source_read_map -crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs GraphResourceKind.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs CliStyle.use_color -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs ColumnMigrationDecision.label -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs MigrationCompatibility.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs MigrationDecision.blocks_build -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs MigrationDecision.checks_compatibility -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs MigrationDecision.label -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs MigrationDiscovery.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs MigrationPromotion.label -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs RetentionDirection.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs RetentionPlanPhase.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs SourceWriteStrategy.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs BackfillAction.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs PlanAction.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs PlanReason.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs PythonIdentityStatus.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs PythonRunPhase.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs SchemaChangeKind.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs SourceWriteStrategy.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs WarningSeverity.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs BackfillAction.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs ColumnMigrationDecision.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs FutureCursorAction.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs MaterializationType.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs MicrobatchLimitAction.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs MigrationCompatibility.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs MigrationDecision.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs MigrationDiscovery.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs MigrationPromotion.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs OldNameViewAction.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs PlanAction.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs PlanReason.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs PythonIdentityStatus.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs PythonRunPhase.value -crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs WarningSeverity.value +crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatches.rs PlanEntryBuildInputs.max_microbatches +crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs PlanEntryBuildInputs.max_microbatches +crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs PlanEntryBuildInputs.max_microbatches_is_override +crates/sqlbuild-python-planner/src/planner_strategy/_helpers/microbatch_limits.rs PlanEntryBuildInputs.microbatch_limit_action +crates/sqlbuild-python-planner/src/planner_strategy/main/microbatch_limits/apply_configured_microbatch_limits.rs PlanEntryBuildInputs.microbatch_limit_action +crates/sqlbuild-python-planner/src/planner_strategy/_helpers/model_entries.rs TableType.value +crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_freshness.rs PlannerRelationsContext.source_read_map +crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs PlannerRelationsContext.listed_source_names +crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/sources/source_tables.rs PlannerRelationsContext.source_read_map +crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs GraphResourceKind.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs CliStyle.use_color +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs ColumnMigrationDecision.label +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs MigrationCompatibility.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs MigrationDecision.blocks_build +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs MigrationDecision.checks_compatibility +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs MigrationDecision.label +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs MigrationDiscovery.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs MigrationPromotion.label +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs RetentionDirection.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/migrations.rs RetentionPlanPhase.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs SourceWriteStrategy.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs BackfillAction.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs PlanAction.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs PlanReason.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs PythonIdentityStatus.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs PythonRunPhase.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs SchemaChangeKind.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs SourceWriteStrategy.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs WarningSeverity.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs BackfillAction.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs ColumnMigrationDecision.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs FutureCursorAction.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs MaterializationType.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs MicrobatchLimitAction.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs MigrationCompatibility.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs MigrationDecision.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs MigrationDiscovery.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs MigrationPromotion.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs OldNameViewAction.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs PlanAction.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs PlanReason.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs PythonIdentityStatus.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs PythonRunPhase.value +crates/sqlbuild-python-planner/src/plan_output/_helpers/values.rs WarningSeverity.value crates/sqlbuild-python-base/src/presentation_text/_helpers/functions.rs CliStyle.use_color crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs MacroArgumentError.detail crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs MacroArgumentError.file_path