diff --git a/Cargo.lock b/Cargo.lock index b76db7f99d..4846066f38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4436,7 +4436,6 @@ name = "sqlbuild-python" version = "0.234.0" dependencies = [ "chrono", - "chrono-tz", "fensu-policy", "pyo3", "rayon", @@ -4444,7 +4443,6 @@ dependencies = [ "rustc-hash", "serde", "serde_json", - "sha2 0.10.9", "signal-hook", "sqlbuild-analysis", "sqlbuild-attachments", @@ -4461,6 +4459,7 @@ dependencies = [ "sqlbuild-pysource", "sqlbuild-python-base", "sqlbuild-python-front", + "sqlbuild-python-project", "sqlbuild-recall", "sqlbuild-refactor", "sqlbuild-render", @@ -4469,7 +4468,6 @@ dependencies = [ "sqlbuild-sqltext", "sqlbuild-state", "sqlclean", - "tokio", "uuid", ] @@ -4527,6 +4525,36 @@ dependencies = [ "uuid", ] +[[package]] +name = "sqlbuild-python-project" +version = "0.0.0" +dependencies = [ + "chrono", + "chrono-tz", + "pyo3", + "serde_json", + "sha2 0.10.9", + "sqlbuild-analysis", + "sqlbuild-attachments", + "sqlbuild-cache", + "sqlbuild-cli", + "sqlbuild-config", + "sqlbuild-core", + "sqlbuild-cursor", + "sqlbuild-discovery", + "sqlbuild-host", + "sqlbuild-model-config", + "sqlbuild-orchestration", + "sqlbuild-project", + "sqlbuild-pysource", + "sqlbuild-python-base", + "sqlbuild-python-front", + "sqlbuild-rules", + "sqlbuild-sqltext", + "tokio", + "uuid", +] + [[package]] name = "sqlbuild-recall" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 6dadb24d1f..23c523a3cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "crates/sqlbuild-cli", "crates/sqlbuild-python-base", "crates/sqlbuild-python-front", + "crates/sqlbuild-python-project", "crates/sqlbuild-python", ] resolver = "2" @@ -59,9 +60,10 @@ order = [ "sqlbuild-cli", "sqlbuild-python-base", "sqlbuild-python-front", + "sqlbuild-python-project", "sqlbuild-python", ] -python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python"] +python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python"] polyglot-floor = "sqlbuild-analysis" [workspace.package] @@ -183,6 +185,7 @@ sqlbuild-process-ownership = { path = "crates/sqlbuild-process-ownership" } sqlbuild-project = { path = "crates/sqlbuild-project" } sqlbuild-python-base = { path = "crates/sqlbuild-python-base" } sqlbuild-python-front = { path = "crates/sqlbuild-python-front" } +sqlbuild-python-project = { path = "crates/sqlbuild-python-project" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } sqlbuild-recall = { path = "crates/sqlbuild-recall" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } diff --git a/crates/README.md b/crates/README.md index 3b6a0de5bd..d6edeeaef9 100644 --- a/crates/README.md +++ b/crates/README.md @@ -26,18 +26,19 @@ tested and benchmarked with `cargo test` alone. | `sqlbuild-analysis` | SQL analysis over polyglot: query analysis, binding, semantic validation and usage, column references, type normalization, FAST/RICH lineage, SQL-test/scenario extraction and compiler-side assembly, and Snowflake cost statement classification and query-tag parsing. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. Its caches and rule fingerprints key on the build identity the binary installs at startup. | | `sqlbuild-refactor` | Native rename/move plans, contract-adoption edits, MODEL/SCHEMA/YAML text edits and atomic staging/commit with rollback. Shared scope placement delegates to the native scope implementation; user Python source inspection remains at the Python AST boundary. | -| `sqlbuild-project` | The compiled project as plain Rust: `CompiledProject` with the producers' facts and each resource's records (model inputs and schemas, compiled models and their analysis, sources, functions, audits and seeds), and what those records answer without Python, such as lineage rows, resolved namespaces and dependency keys. Values the host owns, such as user config and discovered files, and the views it builds lazily live in one slot per record, whose types `sqlbuild-python` supplies through the `ProjectHost` trait; reads that need Python stay in the bindings as extension traits. | +| `sqlbuild-project` | The compiled project as plain Rust: `CompiledProject` with the producers' facts and each resource's records (model inputs and schemas, compiled models and their analysis, sources, functions, audits and seeds), and what those records answer without Python, such as lineage rows, resolved namespaces and dependency keys. Values the host owns, such as user config and discovered files, and the views it builds lazily live in one slot per record, whose types `sqlbuild-python-project` supplies through the `ProjectHost` trait; reads that need Python stay in the bindings as extension traits. | | `sqlbuild-cli` | The `sqb` and `sqlbuild` executables: `--version`, unchanged compiles replayed from the shared reuse store with Python's exact streams, exit code and compute log capture, and a handoff of every other invocation to the installed `sqlbuild-python` script. Built as its own wheel (`crates/sqlbuild-cli/pyproject.toml`). | | `sqlbuild-python-base` | The base of the PyO3 bindings: native record, enum and error classes with their specs, registration, construction and field reads; keyword calls, imports, conversions and panics at the Python boundary; host entry calls; the process allocator; and the shared runtime the binding modules build on: the native store, compile context, diagnostics, progress and terminal text, warehouse access and orchestration reads. | | `sqlbuild-python-front` | The front of the compile pipeline as PyO3 bindings, on top of `sqlbuild-python-base`: project discovery, model configuration and headers, scopes, macro rendering and authored values, SQL text, model attachment and the compile driver, cursors, contract and refactor facts, and the warehouse state tables of consumption, observations and audit results. | -| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-front` and `sqlbuild-python-base`. | +| `sqlbuild-python-project` | The compiled project as PyO3 bindings, on top of `sqlbuild-python-front`: the project host and the Python views of compiled models, inputs and resources, the compiled-project snapshot, analysis classes and the analysis session, the project graph, references, lineage, fingerprints, contracts and semantic shapes, audit rendering, and the planner's identities, scope, selection, phases, fixture strategy and SQL test planning. | +| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. | ## Layering Dependencies point one way, from the top of this graph to the bottom: ```text -sqlbuild-python -> sqlbuild-python-front -> sqlbuild-python-base +sqlbuild-python -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base -> sqlbuild-project -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core diff --git a/crates/sqlbuild-python-front/src/attachments/_helpers/argument_values.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/argument_values.rs index 4b5621ff57..caca7e0466 100644 --- a/crates/sqlbuild-python-front/src/attachments/_helpers/argument_values.rs +++ b/crates/sqlbuild-python-front/src/attachments/_helpers/argument_values.rs @@ -1,59 +1,14 @@ //! Python audit argument values read as plain argument values. -use crate::model_config::main::config::config_object::config_object; -use crate::model_config::types::PyConfigNode; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ - PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString, - PyStringMethods, PyTuple, PyTupleMethods, + PyBool, PyFloat, PyInt, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, + PyTupleMethods, }; use sqlbuild_attachments::audits::models::ArgumentValue; -use sqlbuild_model_config::config_values::models::ConfigNode; - -/// Native-built argument containers need no intermediate Python list or tuple. -pub(crate) fn config_argument_value( - py: Python<'_>, - value: &PyConfigNode, -) -> PyResult { - Ok(match value { - ConfigNode::Null => ArgumentValue::Null, - ConfigNode::Bool(value) => ArgumentValue::Boolean(*value), - ConfigNode::Int(value) => ArgumentValue::Number(value.to_string()), - ConfigNode::Str(value) => ArgumentValue::Text(value.clone()), - ConfigNode::List(items) => ArgumentValue::List( - items - .iter() - .map(|item| config_argument_value(py, item)) - .collect::>()?, - ), - ConfigNode::Tuple(items) => ArgumentValue::Tuple( - items - .iter() - .map(|item| config_argument_value(py, item)) - .collect::>()?, - ), - ConfigNode::Map(_) => ArgumentValue::Opaque, - ConfigNode::Float(_) | ConfigNode::Host(_) => { - return argument_value(&config_object(py, value)?); - } - }) -} - -/// Read a `dict[str, object]` in insertion order, leaving out keys no parameter can name. -pub(crate) fn argument_pairs( - arguments: &Bound<'_, PyDict>, -) -> PyResult> { - let mut pairs: Vec<(String, ArgumentValue)> = Vec::with_capacity(arguments.len()); - for (key, value) in arguments.iter() { - if let Ok(key) = key.downcast::() { - pairs.push((key.to_string_lossy().into_owned(), argument_value(&value)?)); - } - } - Ok(pairs) -} /// One plain value; text with lone surrogates, rejected where it enters a compile, reads lossily. -fn argument_value(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn argument_value(value: &Bound<'_, PyAny>) -> PyResult { if value.is_none() { return Ok(ArgumentValue::Null); } diff --git a/crates/sqlbuild-python-front/src/attachments/_helpers/ceremonial_select.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/ceremonial_select.rs index 9e154442e5..f9756ebd37 100644 --- a/crates/sqlbuild-python-front/src/attachments/_helpers/ceremonial_select.rs +++ b/crates/sqlbuild-python-front/src/attachments/_helpers/ceremonial_select.rs @@ -1,7 +1,7 @@ //! Where a SQL test or scenario body omits its trailing ceremonial `SELECT 1`. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::PyResult; +use pyo3::pyfunction; use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitted_select_offset; use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; @@ -11,7 +11,10 @@ use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; /// The code-point offset to insert `SELECT 1` at, or `None`. #[pyfunction] -fn omitted_ceremonial_select(sql: &str, syntax: LexicalSyntaxInput) -> PyResult> { +pub(crate) fn omitted_ceremonial_select( + sql: &str, + syntax: LexicalSyntaxInput, +) -> PyResult> { let syntax: LexicalSyntax = syntax.into(); compiler_guard(|| { Ok(match omitted_select_offset(sql, &syntax) { @@ -20,8 +23,3 @@ fn omitted_ceremonial_select(sql: &str, syntax: LexicalSyntaxInput) -> PyResult< }) }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(omitted_ceremonial_select, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs index a6222f0c50..c26925d74b 100644 --- a/crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs +++ b/crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs @@ -2,49 +2,31 @@ use crate::attachments::constants::MICROBATCH_END_SENTINEL; use crate::attachments::constants::MICROBATCH_START_SENTINEL; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; +use crate::attachments::main::analysis_cursor_sql::analysis_cursor_sql; +use crate::attachments::main::compile_input_error::compile_input_error; +use crate::attachments::main::validated_model_cursor_sql::validated_model_cursor_sql; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyString, PyStringMethods}; -use pyo3::{Python, pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::sql_macros::models::TextSplice; +use pyo3::{Python, pyfunction}; use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; -use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics_with_spans::replace_intrinsics_with_spans; -use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::validated_model_intrinsics; -use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_base::records::types::ErrorId; -const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; const CURSOR_START: &str = "__cursor_start"; -const CURSOR_START_CALL: &str = "__cursor_start()"; -const CURSOR_END_CALL: &str = "__cursor_end()"; +pub(crate) const CURSOR_START_CALL: &str = "__cursor_start()"; +pub(crate) const CURSOR_END_CALL: &str = "__cursor_end()"; const INTEGER_CURSOR_TYPE: &str = "integer"; const INTEGER_ANALYSIS_BOUND: &str = "CAST(0 AS BIGINT)"; const TIMESTAMP_ANALYSIS_BOUND: &str = "CAST('2000-01-01 00:00:00' AS TIMESTAMP)"; -const MODEL_SQL_CONTEXT: &str = "Model SQL"; -const SQL_CONTEXT: &str = "SQL"; +pub(crate) const MODEL_SQL_CONTEXT: &str = "Model SQL"; +pub(crate) const SQL_CONTEXT: &str = "SQL"; -const RESERVED_CURSOR_MARKERS: [&str; 2] = [MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL]; - -/// The reserved executor bound markers no authored SQL may contain. -pub(crate) fn reserved_cursor_markers() -> Vec { - RESERVED_CURSOR_MARKERS - .iter() - .map(|marker| (*marker).to_owned()) - .collect() -} - -/// A `CompileInputError` carrying `message`. -pub(crate) fn compile_input_error(py: Python<'_>, message: String) -> PyResult { - Ok(PyErr::from_value( - error_class(py, ErrorId::CompileInput)?.call1((message,))?, - )) -} +pub(crate) const RESERVED_CURSOR_MARKERS: [&str; 2] = + [MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL]; /// `text` when it is a string, else `None`. -fn text_value(value: &Bound<'_, PyAny>) -> Option { +pub(crate) fn text_value(value: &Bound<'_, PyAny>) -> Option { match value.downcast::() { Ok(text) => Some(text.to_string_lossy().into_owned()), Err(_) => None, @@ -70,60 +52,8 @@ pub(crate) fn replaced( outcome.or_else(|message| Err(compile_input_error(py, message)?)) } -/// One model query with canonical intrinsic calls, raising when the model may not use them. -pub(crate) fn validated_model_cursor_sql( - py: Python<'_>, - sql: &str, - (model_name, config_values): (&str, &Bound<'_, PyAny>), -) -> PyResult { - let incremental: bool = text_value(&mapping_get(config_values, "materialized")?) - .is_some_and(|materialized| materialized == INCREMENTAL_MATERIALIZATION); - let cursor: Option = text_value(&mapping_get(config_values, "cursor")?); - let reserved_markers: Vec = reserved_cursor_markers(); - let outcome: Result = compiler_guard(|| { - Ok(validated_model_intrinsics( - sql, - &reserved_markers, - &IntrinsicModel { - name: model_name, - incremental, - cursor: cursor.as_deref(), - }, - )) - })?; - outcome.or_else(|message| Err(compile_input_error(py, message)?)) -} - -/// `sql` with intrinsics replaced by stable typed literals, only for static SQL analysis. -pub(crate) fn analysis_cursor_sql( - py: Python<'_>, - sql: &str, - cursor_type: &Bound<'_, PyAny>, -) -> PyResult { - let literal: &str = analysis_bound(text_value(cursor_type).as_deref()); - Ok(replaced(py, sql, MODEL_SQL_CONTEXT, (literal, literal))?.0) -} - -/// `analysis_cursor_sql` with each replaced call as a splice of `sql`. -pub(crate) fn analysis_cursor_splices( - sql: &str, - cursor_type: Option<&str>, -) -> Result<(String, Vec), String> { - let literal: &str = analysis_bound(cursor_type); - let (replaced, spans) = - replace_intrinsics_with_spans(sql, MODEL_SQL_CONTEXT, |_| literal.to_owned())?; - let splices: Vec = spans - .iter() - .map(|span| TextSplice { - source: (span.source_start, span.source_end), - output: (span.output_start, span.output_end), - }) - .collect(); - Ok((replaced, splices)) -} - /// The literal static analysis reads for every cursor intrinsic of a model with `cursor_type`. -fn analysis_bound(cursor_type: Option<&str>) -> &'static str { +pub(crate) fn analysis_bound(cursor_type: Option<&str>) -> &'static str { if cursor_type == Some(INTEGER_CURSOR_TYPE) { INTEGER_ANALYSIS_BOUND } else { @@ -134,7 +64,7 @@ fn analysis_bound(cursor_type: Option<&str>) -> &'static str { /// One model query with canonical intrinsic calls, raising `CompileInputError` when invalid. #[pyfunction] #[pyo3(signature = (*, sql, config_values, model_name))] -fn validated_model_cursor_intrinsics( +pub(crate) fn validated_model_cursor_intrinsics( py: Python<'_>, sql: &str, config_values: &Bound<'_, PyAny>, @@ -146,41 +76,10 @@ fn validated_model_cursor_intrinsics( /// `sql` with intrinsics replaced by typed literals of `cursor_type` for static analysis. #[pyfunction] #[pyo3(signature = (*, sql, cursor_type))] -fn cursor_intrinsics_analysis_sql( +pub(crate) fn cursor_intrinsics_analysis_sql( py: Python<'_>, sql: &str, cursor_type: &Bound<'_, PyAny>, ) -> PyResult { analysis_cursor_sql(py, sql, cursor_type) } - -/// Whether `sql` uses cursor intrinsics, and `sql` with both bounds rendered when given. -#[pyfunction] -#[pyo3(signature = (*, sql, start_sql=None, end_sql=None))] -pub(crate) fn resolve_cursor_intrinsics( - py: Python<'_>, - sql: &str, - start_sql: Option<&str>, - end_sql: Option<&str>, -) -> PyResult<(String, bool)> { - let (rendered, found) = replaced( - py, - sql, - SQL_CONTEXT, - ( - start_sql.unwrap_or(CURSOR_START_CALL), - end_sql.unwrap_or(CURSOR_END_CALL), - ), - )?; - if !found || start_sql.is_none() || end_sql.is_none() { - return Ok((sql.to_owned(), found)); - } - Ok((rendered, true)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(validated_model_cursor_intrinsics, module)?)?; - module.add_function(wrap_pyfunction!(cursor_intrinsics_analysis_sql, module)?)?; - module.add_function(wrap_pyfunction!(resolve_cursor_intrinsics, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/attachments/_helpers/function_headers.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/function_headers.rs index 98c8087ea6..2eeedb15d6 100644 --- a/crates/sqlbuild-python-front/src/attachments/_helpers/function_headers.rs +++ b/crates/sqlbuild-python-front/src/attachments/_helpers/function_headers.rs @@ -5,30 +5,10 @@ use pyo3::types::{ PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, PyTupleMethods, }; -use sqlbuild_attachments::functions::main::parse_function_header::parse_function_header; -use sqlbuild_attachments::functions::models::{FunctionHeader, FunctionLanguage, HeaderValue}; - -/// One function file's parsed header; non-string keys and `None` values are never read. -pub(crate) fn parsed_header( - header_values: &Bound<'_, PyDict>, - language: FunctionLanguage, - relative_path: &str, -) -> FunctionHeader { - let mut header: Vec<(String, HeaderValue)> = Vec::with_capacity(header_values.len()); - for (key, value) in header_values.iter() { - let Some(name) = text_value(&key) else { - continue; - }; - if value.is_none() { - continue; - } - header.push((name, header_value(&value))); - } - parse_function_header(&header, language, relative_path) -} +use sqlbuild_attachments::functions::models::HeaderValue; /// One header value. -fn header_value(value: &Bound<'_, PyAny>) -> HeaderValue { +pub(crate) fn header_value(value: &Bound<'_, PyAny>) -> HeaderValue { if let Some(text) = text_value(value) { return HeaderValue::Text(text); } @@ -50,7 +30,7 @@ fn header_value(value: &Bound<'_, PyAny>) -> HeaderValue { } /// A `str` (or subclass) as text; lone surrogates, rejected at compile entry, read as U+FFFD. -fn text_value(value: &Bound<'_, PyAny>) -> Option { +pub(crate) fn text_value(value: &Bound<'_, PyAny>) -> Option { match value.downcast::() { Ok(text) => Some(text.to_string_lossy().into_owned()), Err(_) => None, diff --git a/crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs b/crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs index 8f151e1d20..029661b7d9 100644 --- a/crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs +++ b/crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs @@ -1,18 +1,10 @@ //! Reject executable SQL that still contains an unresolved reference call. use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyString, PyStringMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_attachments::reference_markers::main::unresolved_reference_marker::unresolved_reference_marker; -use sqlbuild_attachments::sql_lexing::main::unclosed_message::unclosed_message; - -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::types::ErrorId; +use pyo3::prelude::{PyAnyMethods, PyErr, PyResult, Python}; /// The reference function and diagnostic code of each marker kind, in the scanner's order. -const MARKER_KINDS: [(&str, &str); 5] = [ +pub(crate) const MARKER_KINDS: [(&str, &str); 5] = [ ("__ref", "R001"), ("__seed", "R002"), ("__source", "R003"), @@ -21,48 +13,10 @@ const MARKER_KINDS: [(&str, &str); 5] = [ ]; /// A `ValueError` carrying `message` and `code` attributes. -fn coded_value_error(py: Python<'_>, message: String, code: &str) -> PyResult { +pub(crate) fn coded_value_error(py: Python<'_>, message: String, code: &str) -> PyResult { let error: PyErr = PyValueError::new_err(message.clone()); let value = error.value(py); value.setattr("message", message)?; value.setattr("code", code)?; Ok(error) } - -/// Fail if executable SQL still contains unresolved reference markers or an unclosed construct. -#[pyfunction] -#[pyo3(signature = (*, sql, context))] -pub(crate) fn assert_no_unresolved_sql_markers( - py: Python<'_>, - sql: &Bound<'_, PyString>, - context: &str, -) -> PyResult<()> { - let text = sql.to_string_lossy(); - let outcome: (Option, Option) = compiler_guard(|| { - Ok(match unresolved_reference_marker(&text) { - Ok(marker) => (marker, None), - Err(construct) => (None, Some(unclosed_message(context, construct))), - }) - })?; - match outcome { - (_, Some(unclosed)) => Err(PyErr::from_value( - error_class(py, ErrorId::CompileInput)?.call1((unclosed,))?, - )), - (Some(marker), None) => { - let (function, code) = MARKER_KINDS.get(marker).copied().ok_or_else(|| { - PyValueError::new_err(format!("unknown reference marker kind {marker}")) - })?; - Err(coded_value_error( - py, - format!("{context} still contains unresolved {function}() markers"), - code, - )?) - } - (None, None) => Ok(()), - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(assert_no_unresolved_sql_markers, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_splices.rs b/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_splices.rs index f7caf3f420..77a4e6f834 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_splices.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_splices.rs @@ -1,13 +1,24 @@ //! `analysis_cursor_sql` with each replaced call as a splice of `sql`. -use crate::attachments::_helpers::cursor_intrinsics; +use crate::attachments::_helpers::cursor_intrinsics::MODEL_SQL_CONTEXT; +use crate::attachments::_helpers::cursor_intrinsics::analysis_bound; use sqlbuild_analysis::sql_macros::models::TextSplice; +use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics_with_spans::replace_intrinsics_with_spans; /// `analysis_cursor_sql` with each replaced call as a splice of `sql`. -#[inline] pub fn analysis_cursor_splices( sql: &str, cursor_type: Option<&str>, ) -> Result<(String, Vec), String> { - cursor_intrinsics::analysis_cursor_splices(sql, cursor_type) + let literal: &str = analysis_bound(cursor_type); + let (replaced, spans) = + replace_intrinsics_with_spans(sql, MODEL_SQL_CONTEXT, |_| literal.to_owned())?; + let splices: Vec = spans + .iter() + .map(|span| TextSplice { + source: (span.source_start, span.source_end), + output: (span.output_start, span.output_end), + }) + .collect(); + Ok((replaced, splices)) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_sql.rs b/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_sql.rs index 6aa9f8b277..235f6c8234 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_sql.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/analysis_cursor_sql.rs @@ -1,17 +1,18 @@ //! `sql` with intrinsics replaced by stable typed literals, only for static SQL analysis. -use crate::attachments::_helpers::cursor_intrinsics; +use crate::attachments::_helpers::cursor_intrinsics::MODEL_SQL_CONTEXT; +use crate::attachments::_helpers::cursor_intrinsics::analysis_bound; +use crate::attachments::_helpers::cursor_intrinsics::replaced; +use crate::attachments::_helpers::cursor_intrinsics::text_value; use pyo3::Python; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// `sql` with intrinsics replaced by stable typed literals, only for static SQL analysis. -#[inline] pub fn analysis_cursor_sql( py: Python<'_>, sql: &str, cursor_type: &Bound<'_, PyAny>, ) -> PyResult { - cursor_intrinsics::analysis_cursor_sql(py, sql, cursor_type) + let literal: &str = analysis_bound(text_value(cursor_type).as_deref()); + Ok(replaced(py, sql, MODEL_SQL_CONTEXT, (literal, literal))?.0) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/argument_pairs.rs b/crates/sqlbuild-python-front/src/attachments/main/argument_pairs.rs index 9b2119bd47..42d5fb8473 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/argument_pairs.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/argument_pairs.rs @@ -1,13 +1,17 @@ //! Read a `dict[str, object]` in insertion order, leaving out keys no parameter can name. -use crate::attachments::_helpers::argument_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use crate::attachments::_helpers::argument_values::argument_value; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; use sqlbuild_attachments::audits::models::ArgumentValue; /// Read a `dict[str, object]` in insertion order, leaving out keys no parameter can name. -#[inline] pub fn argument_pairs(arguments: &Bound<'_, PyDict>) -> PyResult> { - argument_values::argument_pairs(arguments) + let mut pairs: Vec<(String, ArgumentValue)> = Vec::with_capacity(arguments.len()); + for (key, value) in arguments.iter() { + if let Ok(key) = key.downcast::() { + pairs.push((key.to_string_lossy().into_owned(), argument_value(&value)?)); + } + } + Ok(pairs) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/assert_no_unresolved_sql_markers.rs b/crates/sqlbuild-python-front/src/attachments/main/assert_no_unresolved_sql_markers.rs index 67a209e017..f54e584918 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/assert_no_unresolved_sql_markers.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/assert_no_unresolved_sql_markers.rs @@ -1,17 +1,46 @@ //! Fail if executable SQL still contains unresolved reference markers or an unclosed construct. -use crate::attachments::_helpers::reference_markers; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyString; +use crate::attachments::_helpers::reference_markers::MARKER_KINDS; +use crate::attachments::_helpers::reference_markers::coded_value_error; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyString, PyStringMethods}; +use sqlbuild_attachments::reference_markers::main::unresolved_reference_marker::unresolved_reference_marker; +use sqlbuild_attachments::sql_lexing::main::unclosed_message::unclosed_message; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; /// Fail if executable SQL still contains unresolved reference markers or an unclosed construct. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, sql, context))] pub fn assert_no_unresolved_sql_markers( py: Python<'_>, sql: &Bound<'_, PyString>, context: &str, ) -> PyResult<()> { - reference_markers::assert_no_unresolved_sql_markers(py, sql, context) + let text = sql.to_string_lossy(); + let outcome: (Option, Option) = compiler_guard(|| { + Ok(match unresolved_reference_marker(&text) { + Ok(marker) => (marker, None), + Err(construct) => (None, Some(unclosed_message(context, construct))), + }) + })?; + match outcome { + (_, Some(unclosed)) => Err(PyErr::from_value( + error_class(py, ErrorId::CompileInput)?.call1((unclosed,))?, + )), + (Some(marker), None) => { + let (function, code) = MARKER_KINDS.get(marker).copied().ok_or_else(|| { + PyValueError::new_err(format!("unknown reference marker kind {marker}")) + })?; + Err(coded_value_error( + py, + format!("{context} still contains unresolved {function}() markers"), + code, + )?) + } + (None, None) => Ok(()), + } } diff --git a/crates/sqlbuild-python-front/src/attachments/main/compile_input_error.rs b/crates/sqlbuild-python-front/src/attachments/main/compile_input_error.rs index 72483c1606..8c4a8e715e 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/compile_input_error.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/compile_input_error.rs @@ -1,12 +1,13 @@ //! A `CompileInputError` carrying `message`. -use crate::attachments::_helpers::cursor_intrinsics; use pyo3::Python; -use pyo3::prelude::PyErr; -use pyo3::prelude::PyResult; +use pyo3::prelude::{PyAnyMethods, PyErr, PyResult}; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; /// A `CompileInputError` carrying `message`. -#[inline] pub fn compile_input_error(py: Python<'_>, message: String) -> PyResult { - cursor_intrinsics::compile_input_error(py, message) + Ok(PyErr::from_value( + error_class(py, ErrorId::CompileInput)?.call1((message,))?, + )) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/config_argument_value.rs b/crates/sqlbuild-python-front/src/attachments/main/config_argument_value.rs index e7f68a7202..f8f08cfaa6 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/config_argument_value.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/config_argument_value.rs @@ -1,13 +1,34 @@ //! Native-built argument containers need no intermediate Python list or tuple. -use crate::attachments::_helpers::argument_values; +use crate::attachments::_helpers::argument_values::argument_value; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::types::PyConfigNode; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{PyResult, Python}; use sqlbuild_attachments::audits::models::ArgumentValue; +use sqlbuild_model_config::config_values::models::ConfigNode; /// Native-built argument containers need no intermediate Python list or tuple. -#[inline] pub fn config_argument_value(py: Python<'_>, value: &PyConfigNode) -> PyResult { - argument_values::config_argument_value(py, value) + Ok(match value { + ConfigNode::Null => ArgumentValue::Null, + ConfigNode::Bool(value) => ArgumentValue::Boolean(*value), + ConfigNode::Int(value) => ArgumentValue::Number(value.to_string()), + ConfigNode::Str(value) => ArgumentValue::Text(value.clone()), + ConfigNode::List(items) => ArgumentValue::List( + items + .iter() + .map(|item| config_argument_value(py, item)) + .collect::>()?, + ), + ConfigNode::Tuple(items) => ArgumentValue::Tuple( + items + .iter() + .map(|item| config_argument_value(py, item)) + .collect::>()?, + ), + ConfigNode::Map(_) => ArgumentValue::Opaque, + ConfigNode::Float(_) | ConfigNode::Host(_) => { + return argument_value(&config_object(py, value)?); + } + }) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs b/crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs index ca1c8b5033..d7c6d9fda0 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs @@ -1,17 +1,27 @@ //! One function file's parsed header; non-string keys and `None` values are never read. -use crate::attachments::_helpers::function_headers; -use pyo3::prelude::Bound; -use pyo3::types::PyDict; -use sqlbuild_attachments::functions::models::FunctionHeader; -use sqlbuild_attachments::functions::models::FunctionLanguage; +use crate::attachments::_helpers::function_headers::header_value; +use crate::attachments::_helpers::function_headers::text_value; +use pyo3::prelude::{Bound, PyAnyMethods}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_attachments::functions::main::parse_function_header::parse_function_header; +use sqlbuild_attachments::functions::models::{FunctionHeader, FunctionLanguage, HeaderValue}; /// One function file's parsed header; non-string keys and `None` values are never read. -#[inline] pub fn parsed_header( header_values: &Bound<'_, PyDict>, language: FunctionLanguage, relative_path: &str, ) -> FunctionHeader { - function_headers::parsed_header(header_values, language, relative_path) + let mut header: Vec<(String, HeaderValue)> = Vec::with_capacity(header_values.len()); + for (key, value) in header_values.iter() { + let Some(name) = text_value(&key) else { + continue; + }; + if value.is_none() { + continue; + } + header.push((name, header_value(&value))); + } + parse_function_header(&header, language, relative_path) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/register_ceremonial_select.rs b/crates/sqlbuild-python-front/src/attachments/main/register_ceremonial_select.rs index a84d8eb71e..3a29a82770 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/register_ceremonial_select.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/register_ceremonial_select.rs @@ -1,12 +1,11 @@ //! Register the ceremonial select functions and classes. -use crate::attachments::_helpers::ceremonial_select; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::attachments::_helpers::ceremonial_select::omitted_ceremonial_select; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the ceremonial select functions and classes. -#[inline] pub fn register_ceremonial_select(module: &Bound<'_, PyModule>) -> PyResult<()> { - ceremonial_select::register(module) + module.add_function(wrap_pyfunction!(omitted_ceremonial_select, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs b/crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs index 4efba4d901..88076527c0 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs @@ -1,12 +1,15 @@ //! Register the cursor intrinsics functions and classes. -use crate::attachments::_helpers::cursor_intrinsics; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::attachments::_helpers::cursor_intrinsics::cursor_intrinsics_analysis_sql; +use crate::attachments::_helpers::cursor_intrinsics::validated_model_cursor_intrinsics; +use crate::attachments::main::resolve_cursor_intrinsics::resolve_cursor_intrinsics; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the cursor intrinsics functions and classes. -#[inline] pub fn register_cursor_intrinsics(module: &Bound<'_, PyModule>) -> PyResult<()> { - cursor_intrinsics::register(module) + module.add_function(wrap_pyfunction!(validated_model_cursor_intrinsics, module)?)?; + module.add_function(wrap_pyfunction!(cursor_intrinsics_analysis_sql, module)?)?; + module.add_function(wrap_pyfunction!(resolve_cursor_intrinsics, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/register_reference_markers.rs b/crates/sqlbuild-python-front/src/attachments/main/register_reference_markers.rs index 9f79437e34..9e9a25e353 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/register_reference_markers.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/register_reference_markers.rs @@ -1,12 +1,11 @@ //! Register the reference markers functions and classes. -use crate::attachments::_helpers::reference_markers; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::attachments::main::assert_no_unresolved_sql_markers::assert_no_unresolved_sql_markers; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the reference markers functions and classes. -#[inline] pub fn register_reference_markers(module: &Bound<'_, PyModule>) -> PyResult<()> { - reference_markers::register(module) + module.add_function(wrap_pyfunction!(assert_no_unresolved_sql_markers, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/reserved_cursor_markers.rs b/crates/sqlbuild-python-front/src/attachments/main/reserved_cursor_markers.rs index 1b3973bc25..cd44c68744 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/reserved_cursor_markers.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/reserved_cursor_markers.rs @@ -1,9 +1,11 @@ //! The reserved executor bound markers no authored SQL may contain. -use crate::attachments::_helpers::cursor_intrinsics; +use crate::attachments::_helpers::cursor_intrinsics::RESERVED_CURSOR_MARKERS; /// The reserved executor bound markers no authored SQL may contain. -#[inline] pub fn reserved_cursor_markers() -> Vec { - cursor_intrinsics::reserved_cursor_markers() + RESERVED_CURSOR_MARKERS + .iter() + .map(|marker| (*marker).to_owned()) + .collect() } diff --git a/crates/sqlbuild-python-front/src/attachments/main/resolve_cursor_intrinsics.rs b/crates/sqlbuild-python-front/src/attachments/main/resolve_cursor_intrinsics.rs index 752c0d4d25..a6b1663486 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/resolve_cursor_intrinsics.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/resolve_cursor_intrinsics.rs @@ -1,16 +1,32 @@ //! Whether `sql` uses cursor intrinsics, and `sql` with both bounds rendered when given. -use crate::attachments::_helpers::cursor_intrinsics; -use pyo3::Python; +use crate::attachments::_helpers::cursor_intrinsics::CURSOR_END_CALL; +use crate::attachments::_helpers::cursor_intrinsics::CURSOR_START_CALL; +use crate::attachments::_helpers::cursor_intrinsics::SQL_CONTEXT; +use crate::attachments::_helpers::cursor_intrinsics::replaced; use pyo3::prelude::PyResult; +use pyo3::{Python, pyfunction}; /// Whether `sql` uses cursor intrinsics, and `sql` with both bounds rendered when given. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, sql, start_sql=None, end_sql=None))] pub fn resolve_cursor_intrinsics( py: Python<'_>, sql: &str, start_sql: Option<&str>, end_sql: Option<&str>, ) -> PyResult<(String, bool)> { - cursor_intrinsics::resolve_cursor_intrinsics(py, sql, start_sql, end_sql) + let (rendered, found) = replaced( + py, + sql, + SQL_CONTEXT, + ( + start_sql.unwrap_or(CURSOR_START_CALL), + end_sql.unwrap_or(CURSOR_END_CALL), + ), + )?; + if !found || start_sql.is_none() || end_sql.is_none() { + return Ok((sql.to_owned(), found)); + } + Ok((rendered, true)) } diff --git a/crates/sqlbuild-python-front/src/attachments/main/validated_model_cursor_sql.rs b/crates/sqlbuild-python-front/src/attachments/main/validated_model_cursor_sql.rs index e93c8a78c7..3214f4162c 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/validated_model_cursor_sql.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/validated_model_cursor_sql.rs @@ -1,17 +1,36 @@ //! One model query with canonical intrinsic calls, raising when the model may not use them. -use crate::attachments::_helpers::cursor_intrinsics; +use crate::attachments::_helpers::cursor_intrinsics::INCREMENTAL_MATERIALIZATION; +use crate::attachments::_helpers::cursor_intrinsics::text_value; +use crate::attachments::main::compile_input_error::compile_input_error; +use crate::attachments::main::reserved_cursor_markers::reserved_cursor_markers; use pyo3::Python; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::validated_model_intrinsics; +use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; /// One model query with canonical intrinsic calls, raising when the model may not use them. -#[inline] pub fn validated_model_cursor_sql( py: Python<'_>, sql: &str, (model_name, config_values): (&str, &Bound<'_, PyAny>), ) -> PyResult { - cursor_intrinsics::validated_model_cursor_sql(py, sql, (model_name, config_values)) + let incremental: bool = text_value(&mapping_get(config_values, "materialized")?) + .is_some_and(|materialized| materialized == INCREMENTAL_MATERIALIZATION); + let cursor: Option = text_value(&mapping_get(config_values, "cursor")?); + let reserved_markers: Vec = reserved_cursor_markers(); + let outcome: Result = compiler_guard(|| { + Ok(validated_model_intrinsics( + sql, + &reserved_markers, + &IntrinsicModel { + name: model_name, + incremental, + cursor: cursor.as_deref(), + }, + )) + })?; + outcome.or_else(|message| Err(compile_input_error(py, message)?)) } diff --git a/crates/sqlbuild-python-front/src/authored_values/_helpers/fields.rs b/crates/sqlbuild-python-front/src/authored_values/_helpers/fields.rs index 3c31b6c879..0a3fc1e0c5 100644 --- a/crates/sqlbuild-python-front/src/authored_values/_helpers/fields.rs +++ b/crates/sqlbuild-python-front/src/authored_values/_helpers/fields.rs @@ -1,8 +1,8 @@ //! Optional authored string and boolean settings, rejected with the caller's error class. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; +use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyString, PyStringMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::models::Request; @@ -42,7 +42,9 @@ impl<'py> Setting<'py> { /// An optional non-empty string setting, or `None` when absent. #[pyfunction] #[pyo3(signature = (**request))] -fn optional_named_string<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn optional_named_string<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let setting: Setting<'py> = Setting::read(&Request::new("optional_named_string", request)?)?; if setting.raw_value.is_none() { return Ok(setting.raw_value); @@ -60,7 +62,9 @@ fn optional_named_string<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult< /// An optional boolean setting, or `default` when absent. #[pyfunction] #[pyo3(signature = (**request))] -fn optional_named_bool<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn optional_named_bool<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("optional_named_bool", request)?; let setting: Setting<'py> = Setting::read(&request)?; if setting.raw_value.is_none() { @@ -71,9 +75,3 @@ fn optional_named_bool<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult) -> PyResult<()> { - module.add_function(wrap_pyfunction!(optional_named_string, module)?)?; - module.add_function(wrap_pyfunction!(optional_named_bool, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/authored_values/_helpers/project_vars.rs b/crates/sqlbuild-python-front/src/authored_values/_helpers/project_vars.rs index 605701b73f..730cb3eb5e 100644 --- a/crates/sqlbuild-python-front/src/authored_values/_helpers/project_vars.rs +++ b/crates/sqlbuild-python-front/src/authored_values/_helpers/project_vars.rs @@ -3,11 +3,8 @@ use sqlbuild_host::entries::models::FacadeHost; use sqlbuild_python_base::native_host::main::host_error::host_error; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyString, PyTypeMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyDict; const JSON_MODULE: &str = "json"; /// Characters of a structured value's JSON preview before it is cut short. @@ -15,12 +12,12 @@ const PREVIEW_LIMIT: usize = 120; const PREVIEW_ELLIPSIS: &str = "..."; /// `SharedInputError(message)`. -fn shared_input_error(py: Python<'_>, message: String) -> PyErr { +pub(crate) fn shared_input_error(py: Python<'_>, message: String) -> PyErr { host_error(py, FacadeHost::SharedInputError, &message) } /// Compact, key-sorted JSON of `value`, cut to the preview limit. -fn preview_json(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn preview_json(value: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = value.py(); let keywords: Bound<'_, PyDict> = PyDict::new(py); keywords.set_item("separators", (",", ":"))?; @@ -39,58 +36,3 @@ fn preview_json(value: &Bound<'_, PyAny>) -> PyResult { .collect(); Ok(format!("{kept}{PREVIEW_ELLIPSIS}")) } - -/// One project var value as interpolated text; structured values are rejected. -#[pyfunction] -#[pyo3(signature = (*, value, label))] -pub(crate) fn render_project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { - project_var_text(value, label) -} - -/// One project var value as interpolated text; structured values are rejected. -pub(crate) fn project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { - let py: Python<'_> = value.py(); - if value.is_none() { - return Ok(String::new()); - } - if value.is_instance_of::() { - return Ok(if value.is_truthy()? { "true" } else { "false" }.to_owned()); - } - if value.is_instance_of::() - || value.is_instance_of::() - || value.is_instance_of::() - { - return value.str()?.extract(); - } - let structure: Option<&str> = if value.is_instance_of::() { - Some("an object") - } else if value.is_instance_of::() { - Some("an array") - } else { - None - }; - let message: String = match structure { - Some(structure) => format!( - "{label} is {structure} and cannot be interpolated as text: {}. Use a macro to \ - consume structured vars.", - preview_json(value)? - ), - None => format!( - "{label} has unsupported type {} and cannot be interpolated as text", - value.get_type().name()?.repr()? - ), - }; - Err(shared_input_error(py, message)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(render_project_var_text, module)?)?; - Ok(()) -} - -/// `render_project_var_text` as a Python function. -pub(crate) fn render_project_var_text_function( - py: pyo3::Python<'_>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(render_project_var_text, py) -} diff --git a/crates/sqlbuild-python-front/src/authored_values/_helpers/python_nodes.rs b/crates/sqlbuild-python-front/src/authored_values/_helpers/python_nodes.rs index d6f912de6c..21df75d7b3 100644 --- a/crates/sqlbuild-python-front/src/authored_values/_helpers/python_nodes.rs +++ b/crates/sqlbuild-python-front/src/authored_values/_helpers/python_nodes.rs @@ -1,14 +1,12 @@ //! Python-node authoring checks: retry policies, audit cases and hook reads, in Python's operators. use pyo3::exceptions::{PyBaseException, PyTypeError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{ PyDict, PyDictMethods, PyInt, PyMemoryView, PyString, PyStringMethods, PyTuple, PyType, }; use pyo3::types::{PyTupleMethods, PyTypeMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::identifier_text::main::is_lowercase_identifier::is_lowercase_identifier; use sqlbuild_host::entries::models::{FacadeHost, UserCodeHost}; use sqlbuild_model_config::header_metadata::main::json_meta_problem::json_meta_problem; @@ -57,7 +55,7 @@ fn exception_class(value: &Bound<'_, PyAny>) -> PyResult { /// The validated `retry_on` exception classes of a retry policy. #[pyfunction] #[pyo3(signature = (**request))] -fn retry_policy_exceptions<'py>( +pub(crate) fn retry_policy_exceptions<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request: Request<'_, 'py> = Request::new("retry_policy_exceptions", request)?; @@ -100,7 +98,7 @@ fn retry_policy_exceptions<'py>( /// The delay before retry `retry_index`: exponential backoff, capped, and jittered by `random`. #[pyfunction] #[pyo3(signature = (*, retry_policy, retry_index))] -fn retry_delay_seconds<'py>( +pub(crate) fn retry_delay_seconds<'py>( retry_policy: &Bound<'py, PyAny>, retry_index: &Bound<'py, PyAny>, ) -> PyResult> { @@ -145,7 +143,7 @@ fn valid_count(value: &Bound<'_, PyAny>) -> PyResult { /// Reject an audit case without a lowercase name, a definition, valid counts, or JSON meta. #[pyfunction] #[pyo3(signature = (**request))] -fn validate_audit_case(request: Option<&Bound<'_, PyDict>>) -> Result<(), PyErr> { +pub(crate) fn validate_audit_case(request: Option<&Bound<'_, PyDict>>) -> Result<(), PyErr> { let request = Request::new("validate_audit_case", request)?; let py: Python<'_> = request.py(); let name: &Bound<'_, PyAny> = &request.get("name")?; @@ -179,7 +177,9 @@ fn validate_audit_case(request: Option<&Bound<'_, PyDict>>) -> Result<(), PyErr> /// `@hook(reads=...)` references in first-seen order; a leading invalid `None` is tolerated. #[pyfunction] -fn normalize_hook_reads<'py>(value: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn normalize_hook_reads<'py>( + value: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = value.py(); let reference: Bound<'py, PyAny> = host_entry(py, UserCodeHost::SqlResourceRef)?; let reads: Vec> = if value.is_instance(&reference)? { @@ -209,11 +209,3 @@ fn normalize_hook_reads<'py>(value: &Bound<'py, PyAny>) -> PyResult) -> PyResult<()> { - module.add_function(wrap_pyfunction!(retry_policy_exceptions, module)?)?; - module.add_function(wrap_pyfunction!(retry_delay_seconds, module)?)?; - module.add_function(wrap_pyfunction!(validate_audit_case, module)?)?; - module.add_function(wrap_pyfunction!(normalize_hook_reads, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs b/crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs index b0be7fdc1d..3ffdb4e9aa 100644 --- a/crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs +++ b/crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs @@ -1,22 +1,19 @@ //! Typed SQL value normalization between Python authored values and native `SqlValue`s. +use crate::authored_values::main::sql_value::sql_value; +use crate::authored_values::main::sql_value_error::sql_value_error; use crate::authored_values::models::SqlValueClasses; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{ PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, }; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_core::models::{SqlLogicalType, SqlValue}; -use sqlbuild_model_config::sql_values::errors::SqlValueFailure; use sqlbuild_model_config::sql_values::main::normalize_sql_value::normalize_sql_value as normalized; use sqlbuild_model_config::sql_values::models::{RawKey, RawSqlValue, SqlValueLimits}; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::registry::native_attr::native_attr; -use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; fn is_exactly(value: &Bound<'_, PyAny>, class: &Bound<'_, PyAny>) -> bool { @@ -110,7 +107,10 @@ pub(crate) fn logical_type<'py>( classes.logical_type.call1((kind, element)) } -fn payload<'py>(value: &SqlValue, classes: &SqlValueClasses<'py>) -> PyResult> { +pub(crate) fn payload<'py>( + value: &SqlValue, + classes: &SqlValueClasses<'py>, +) -> PyResult> { let py = classes.kind.py(); Ok(match value { SqlValue::Null => py.None().into_bound(py), @@ -140,35 +140,10 @@ fn payload<'py>(value: &SqlValue, classes: &SqlValueClasses<'py>) -> PyResult( - value: &SqlValue, - classes: &SqlValueClasses<'py>, -) -> PyResult> { - classes.sql_value.call1(( - logical_type(&value.logical_type(), classes)?, - payload(value, classes)?, - )) -} - -/// The exception a normalization failure raises. -pub(crate) fn sql_value_error(py: Python<'_>, failure: SqlValueFailure) -> PyResult { - Ok(match failure { - SqlValueFailure::Invalid(message) => PyErr::from_value( - native_attr(py, NativeAttr::SqlValueValidationError)?.call1((message,))?, - ), - SqlValueFailure::DecimalOverflow => { - let overflow = py.import("decimal")?.getattr("Overflow")?; - let classes = pyo3::types::PyList::new(py, [overflow.clone()])?; - PyErr::from_value(overflow.call1((classes,))?) - } - }) -} - /// `normalize_sql_value`: the canonical `SqlValue` of an authored value. #[pyfunction] #[pyo3(signature = (*, raw_value, context, explicit_type = None, limits = None))] -fn normalize_sql_value<'py>( +pub(crate) fn normalize_sql_value<'py>( raw_value: &Bound<'py, PyAny>, context: &str, explicit_type: Option<&str>, @@ -191,7 +166,3 @@ fn normalize_sql_value<'py>( Err(failure) => Err(sql_value_error(py, failure)?), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(normalize_sql_value, module)?) -} diff --git a/crates/sqlbuild-python-front/src/authored_values/main/project_var_text.rs b/crates/sqlbuild-python-front/src/authored_values/main/project_var_text.rs index 81300b47ea..8f92f84bb9 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/project_var_text.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/project_var_text.rs @@ -1,12 +1,42 @@ //! One project var value as interpolated text; structured values are rejected. -use crate::authored_values::_helpers::project_vars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::authored_values::_helpers::project_vars::preview_json; +use crate::authored_values::_helpers::project_vars::shared_input_error; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyString, PyTypeMethods}; /// One project var value as interpolated text; structured values are rejected. -#[inline] pub fn project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { - project_vars::project_var_text(value, label) + let py: Python<'_> = value.py(); + if value.is_none() { + return Ok(String::new()); + } + if value.is_instance_of::() { + return Ok(if value.is_truthy()? { "true" } else { "false" }.to_owned()); + } + if value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + { + return value.str()?.extract(); + } + let structure: Option<&str> = if value.is_instance_of::() { + Some("an object") + } else if value.is_instance_of::() { + Some("an array") + } else { + None + }; + let message: String = match structure { + Some(structure) => format!( + "{label} is {structure} and cannot be interpolated as text: {}. Use a macro to \ + consume structured vars.", + preview_json(value)? + ), + None => format!( + "{label} has unsupported type {} and cannot be interpolated as text", + value.get_type().name()?.repr()? + ), + }; + Err(shared_input_error(py, message)) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/raw_value.rs b/crates/sqlbuild-python-front/src/authored_values/main/raw_value.rs index 1415e788e2..c244fd23ec 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/raw_value.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/raw_value.rs @@ -2,9 +2,7 @@ use crate::authored_values::_helpers::sql_values; use crate::authored_values::models::SqlValueClasses; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_model_config::sql_values::models::RawSqlValue; /// An authored Python value read by exact type, as `normalize_sql_value` dispatches. diff --git a/crates/sqlbuild-python-front/src/authored_values/main/register_fields.rs b/crates/sqlbuild-python-front/src/authored_values/main/register_fields.rs index a47e495da2..b1680e2e8f 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/register_fields.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/register_fields.rs @@ -1,12 +1,13 @@ //! Register the fields functions and classes. -use crate::authored_values::_helpers::fields; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::authored_values::_helpers::fields::optional_named_bool; +use crate::authored_values::_helpers::fields::optional_named_string; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the fields functions and classes. -#[inline] pub fn register_fields(module: &Bound<'_, PyModule>) -> PyResult<()> { - fields::register(module) + module.add_function(wrap_pyfunction!(optional_named_string, module)?)?; + module.add_function(wrap_pyfunction!(optional_named_bool, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/register_project_vars.rs b/crates/sqlbuild-python-front/src/authored_values/main/register_project_vars.rs index 52821b1733..a1ce916e50 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/register_project_vars.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/register_project_vars.rs @@ -1,12 +1,11 @@ //! Register the project vars functions and classes. -use crate::authored_values::_helpers::project_vars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::authored_values::main::render_project_var_text::render_project_var_text; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the project vars functions and classes. -#[inline] pub fn register_project_vars(module: &Bound<'_, PyModule>) -> PyResult<()> { - project_vars::register(module) + module.add_function(wrap_pyfunction!(render_project_var_text, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs b/crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs index 7db72c6561..846d622ee1 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs @@ -1,12 +1,17 @@ //! Register the python nodes functions and classes. -use crate::authored_values::_helpers::python_nodes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::authored_values::_helpers::python_nodes::normalize_hook_reads; +use crate::authored_values::_helpers::python_nodes::retry_delay_seconds; +use crate::authored_values::_helpers::python_nodes::retry_policy_exceptions; +use crate::authored_values::_helpers::python_nodes::validate_audit_case; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the python nodes functions and classes. -#[inline] pub fn register_python_nodes(module: &Bound<'_, PyModule>) -> PyResult<()> { - python_nodes::register(module) + module.add_function(wrap_pyfunction!(retry_policy_exceptions, module)?)?; + module.add_function(wrap_pyfunction!(retry_delay_seconds, module)?)?; + module.add_function(wrap_pyfunction!(validate_audit_case, module)?)?; + module.add_function(wrap_pyfunction!(normalize_hook_reads, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/register_sql_values.rs b/crates/sqlbuild-python-front/src/authored_values/main/register_sql_values.rs index 07a16cc1d4..2de7dbd65e 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/register_sql_values.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/register_sql_values.rs @@ -1,12 +1,10 @@ //! Register the sql values functions and classes. -use crate::authored_values::_helpers::sql_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::authored_values::_helpers::sql_values::normalize_sql_value; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the sql values functions and classes. -#[inline] pub fn register_sql_values(module: &Bound<'_, PyModule>) -> PyResult<()> { - sql_values::register(module) + module.add_function(wrap_pyfunction!(normalize_sql_value, module)?) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text.rs b/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text.rs index d3a2aac8df..7dce4c2620 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text.rs @@ -1,12 +1,12 @@ //! One project var value as interpolated text; structured values are rejected. -use crate::authored_values::_helpers::project_vars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::authored_values::main::project_var_text::project_var_text; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// One project var value as interpolated text; structured values are rejected. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, value, label))] pub fn render_project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { - project_vars::render_project_var_text(value, label) + project_var_text(value, label) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text_function.rs b/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text_function.rs index 9cfdde9bd9..d220e241d4 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text_function.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/render_project_var_text_function.rs @@ -1,11 +1,10 @@ //! `render_project_var_text` as a Python function. -use crate::authored_values::_helpers::project_vars; +use crate::authored_values::main::render_project_var_text::render_project_var_text; /// `render_project_var_text` as a Python function. -#[inline] pub fn render_project_var_text_function( py: pyo3::Python<'_>, ) -> pyo3::PyResult> { - project_vars::render_project_var_text_function(py) + pyo3::wrap_pyfunction!(render_project_var_text, py) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/sql_value.rs b/crates/sqlbuild-python-front/src/authored_values/main/sql_value.rs index 24b7318920..1661b9edd9 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/sql_value.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/sql_value.rs @@ -1,17 +1,18 @@ //! A Python `SqlValue` for a native one. -use crate::authored_values::_helpers::sql_values; +use crate::authored_values::_helpers::sql_values::logical_type; +use crate::authored_values::_helpers::sql_values::payload; use crate::authored_values::models::SqlValueClasses; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_core::models::SqlValue; /// A Python `SqlValue` for a native one. -#[inline] pub fn sql_value<'py>( value: &SqlValue, classes: &SqlValueClasses<'py>, ) -> PyResult> { - sql_values::sql_value(value, classes) + classes.sql_value.call1(( + logical_type(&value.logical_type(), classes)?, + payload(value, classes)?, + )) } diff --git a/crates/sqlbuild-python-front/src/authored_values/main/sql_value_error.rs b/crates/sqlbuild-python-front/src/authored_values/main/sql_value_error.rs index a3fbc75484..33ee056f1b 100644 --- a/crates/sqlbuild-python-front/src/authored_values/main/sql_value_error.rs +++ b/crates/sqlbuild-python-front/src/authored_values/main/sql_value_error.rs @@ -1,13 +1,20 @@ //! The exception a normalization failure raises. -use crate::authored_values::_helpers::sql_values; -use pyo3::prelude::PyErr; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{PyAnyMethods, PyErr, PyResult, Python}; use sqlbuild_model_config::sql_values::errors::SqlValueFailure; +use sqlbuild_python_base::records::main::registry::native_attr::native_attr; +use sqlbuild_python_base::records::types::NativeAttr; /// The exception a normalization failure raises. -#[inline] pub fn sql_value_error(py: Python<'_>, failure: SqlValueFailure) -> PyResult { - sql_values::sql_value_error(py, failure) + Ok(match failure { + SqlValueFailure::Invalid(message) => PyErr::from_value( + native_attr(py, NativeAttr::SqlValueValidationError)?.call1((message,))?, + ), + SqlValueFailure::DecimalOverflow => { + let overflow = py.import("decimal")?.getattr("Overflow")?; + let classes = pyo3::types::PyList::new(py, [overflow.clone()])?; + PyErr::from_value(overflow.call1((classes,))?) + } + }) } diff --git a/crates/sqlbuild-python-front/src/command_runtime/_helpers/concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/_helpers/concurrency.rs index 41f7b25158..9f5038fdf0 100644 --- a/crates/sqlbuild-python-front/src/command_runtime/_helpers/concurrency.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/_helpers/concurrency.rs @@ -1,61 +1,3 @@ //! The worker limit of one command: `--concurrency`, the target, then `[settings]`. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_config::effective::main::effective_concurrency::effective_concurrency; -use sqlbuild_config::effective::models::{ConcurrencyLayers, EffectiveConcurrency}; - -use crate::compile_attachment::main::build_effective_settings::build_effective_settings; -use crate::contracts::main::resolve_target_config::resolve_target_config; -use crate::contracts::main::resolve_target_name::resolve_target_name; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -const CONCURRENCY: &str = "concurrency"; - -/// The effective concurrency of a command on `selected_target`, and the layer that set it. -pub(crate) fn resolve_concurrency( - requested: &Bound<'_, PyAny>, - discovered: &Bound<'_, PyAny>, - selected_target: &Bound<'_, PyAny>, -) -> PyResult { - resolve_configured_concurrency( - requested, - ( - &native_field(discovered, "project_config")?, - &native_field(discovered, "local_config")?, - ), - selected_target, - ) -} - -/// `resolve_concurrency` from the project and local configs themselves. -pub(crate) fn resolve_configured_concurrency( - requested: &Bound<'_, PyAny>, - (project_config, local_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - selected_target: &Bound<'_, PyAny>, -) -> PyResult { - let py = project_config.py(); - let target: Option<(String, i64)> = - match resolve_target_name(py, project_config, local_config, selected_target)? { - Some(name) => { - let config: Bound<'_, PyAny> = - resolve_target_config(py, project_config, local_config, name.as_any())?; - let limit: Option = native_field(&config, CONCURRENCY)?.extract()?; - limit.map(|limit| Ok::<_, pyo3::PyErr>((name.extract::()?, limit))) - } - None => None, - } - .transpose()?; - let settings: i64 = native_field( - &build_effective_settings(project_config, local_config)?, - CONCURRENCY, - )? - .extract()?; - let local_override: bool = - native_field(local_config, "setting_overrides")?.contains(CONCURRENCY)?; - Ok(effective_concurrency(&ConcurrencyLayers { - requested: requested.extract()?, - target: target.as_ref().map(|(name, limit)| (name.as_str(), *limit)), - settings, - local_override, - })) -} +pub(crate) const CONCURRENCY: &str = "concurrency"; diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs index 0c69f9bdda..da83e1347d 100644 --- a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs @@ -1,17 +1,22 @@ //! The effective concurrency of a command on `selected_target`, and the layer that set it. -use crate::command_runtime::_helpers::concurrency; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::command_runtime::main::resolve_configured_concurrency::resolve_configured_concurrency; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_config::effective::models::EffectiveConcurrency; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The effective concurrency of a command on `selected_target`, and the layer that set it. -#[inline] pub fn resolve_concurrency( requested: &Bound<'_, PyAny>, discovered: &Bound<'_, PyAny>, selected_target: &Bound<'_, PyAny>, ) -> PyResult { - concurrency::resolve_concurrency(requested, discovered, selected_target) + resolve_configured_concurrency( + requested, + ( + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, + ), + selected_target, + ) } diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs index 5faa1fac25..5a40e5d3a8 100644 --- a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs @@ -1,21 +1,43 @@ //! `resolve_concurrency` from the project and local configs themselves. -use crate::command_runtime::_helpers::concurrency; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use sqlbuild_config::effective::models::EffectiveConcurrency; +use crate::command_runtime::_helpers::concurrency::CONCURRENCY; +use crate::compile_attachment::main::build_effective_settings::build_effective_settings; +use crate::contracts::main::resolve_target_config::resolve_target_config; +use crate::contracts::main::resolve_target_name::resolve_target_name; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_config::effective::main::effective_concurrency::effective_concurrency; +use sqlbuild_config::effective::models::{ConcurrencyLayers, EffectiveConcurrency}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `resolve_concurrency` from the project and local configs themselves. -#[inline] pub fn resolve_configured_concurrency( requested: &Bound<'_, PyAny>, (project_config, local_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), selected_target: &Bound<'_, PyAny>, ) -> PyResult { - concurrency::resolve_configured_concurrency( - requested, - (project_config, local_config), - selected_target, - ) + let py = project_config.py(); + let target: Option<(String, i64)> = + match resolve_target_name(py, project_config, local_config, selected_target)? { + Some(name) => { + let config: Bound<'_, PyAny> = + resolve_target_config(py, project_config, local_config, name.as_any())?; + let limit: Option = native_field(&config, CONCURRENCY)?.extract()?; + limit.map(|limit| Ok::<_, pyo3::PyErr>((name.extract::()?, limit))) + } + None => None, + } + .transpose()?; + let settings: i64 = native_field( + &build_effective_settings(project_config, local_config)?, + CONCURRENCY, + )? + .extract()?; + let local_override: bool = + native_field(local_config, "setting_overrides")?.contains(CONCURRENCY)?; + Ok(effective_concurrency(&ConcurrencyLayers { + requested: requested.extract()?, + target: target.as_ref().map(|(name, limit)| (name.as_str(), *limit)), + settings, + local_override, + })) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs b/crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs index 30d675df80..a3b8cd8894 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs @@ -1,48 +1,21 @@ //! The effective connection, settings and vars of a project, its local config and target. +use crate::compile_attachment::main::build_effective_vars::build_effective_vars; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::pyfunction; -use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::types::PyDict; use sqlbuild_model_config::templates::models::TemplateOptions; -use crate::contracts::main::effective_config_settings::effective_config_settings; -use crate::contracts::main::effective_config_vars::effective_config_vars; use crate::contracts::main::effective_connection_fields::effective_connection_fields; -use crate::contracts::main::resolve_target_config::resolve_target_config; -use crate::contracts::main::resolve_target_name::resolve_target_name; -use crate::model_config::main::config::expanded_effective_vars::expanded_effective_vars; use crate::model_config::main::config::expanded_template_data::expanded_template_data; -use crate::model_config::main::config::run_id::run_id; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_base::records::types::RecordId; const CONNECTION_LABEL: &str = "effective connection"; -/// Merge base project connection with the selected environment overrides. -#[pyfunction] -#[pyo3(signature = (*, project_config, local_config, target_config, effective_vars))] -pub(crate) fn build_effective_connection<'py>( - project_config: &Bound<'py, PyAny>, - local_config: &Bound<'py, PyAny>, - target_config: &Bound<'py, PyAny>, - effective_vars: &Bound<'py, PyAny>, -) -> PyResult> { - expanded_connection( - (project_config, local_config), - target_config, - effective_vars, - false, - ) -} - /// Expand only the connection selected for the caller's writer or catalog role. -fn expanded_connection<'py>( +pub(crate) fn expanded_connection<'py>( (project_config, local_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), target_config: &Bound<'py, PyAny>, effective_vars: &Bound<'py, PyAny>, @@ -72,77 +45,8 @@ fn expanded_connection<'py>( ) } -/// Merge project settings with local developer overrides. -#[pyfunction] -#[pyo3(signature = (*, project_config, local_config))] -pub(crate) fn build_effective_settings<'py>( - project_config: &Bound<'py, PyAny>, - local_config: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_config.py(); - let settings_class: Bound<'py, PyAny> = record_class(py, RecordId::SettingsConfig)?; - let names: Vec> = imported(py, "dataclasses", "fields")? - .call1((&settings_class,))? - .try_iter()? - .map(|field| native_field(&field?, "name")) - .collect::>()?; - let values: Bound<'py, PyDict> = effective_config_settings( - py, - ( - native_field(project_config, "settings")?, - native_field(local_config, "settings")?, - ), - &native_field(local_config, "setting_overrides")?, - names - .into_iter() - .map(|name| Ok(name.cast_into::()?)) - .collect::>()?, - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? - .downcast_into()?; - new_record(py, RecordId::SettingsConfig, (), Some(&values)) -} - -/// Merge effective vars using the locked precedence order. -#[pyfunction] -#[pyo3(signature = (*, project_config, local_config, target_config, cli_vars))] -pub(crate) fn build_effective_vars<'py>( - project_config: &Bound<'py, PyAny>, - local_config: &Bound<'py, PyAny>, - target_config: &Bound<'py, PyAny>, - cli_vars: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_config.py(); - let merged: Bound<'py, PyAny> = effective_config_vars( - py, - (project_config.clone(), local_config.clone()), - target_config, - cli_vars, - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; - expanded_effective_vars(merged.downcast()?) -} - -/// The selected target name of a discovered project and its effective config, if any. -pub(crate) fn target_selection<'py>( - discovered: &Bound<'py, PyAny>, - selected: Option>, -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - let py: Python<'py> = discovered.py(); - let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; - let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; - let selected: Bound<'py, PyAny> = selected.unwrap_or_else(|| py.None().into_bound(py)); - let Some(name) = resolve_target_name(py, &project_config, &local_config, &selected)? else { - return Ok((py.None().into_bound(py), py.None().into_bound(py))); - }; - let name: Bound<'py, PyAny> = name.into_any(); - let config: Bound<'py, PyAny> = - resolve_target_config(py, &project_config, &local_config, &name)?; - Ok((name, config)) -} - /// The effective vars of a discovered project for `target_config` and optional CLI vars. -fn discovered_vars<'py>( +pub(crate) fn discovered_vars<'py>( discovered: &Bound<'py, PyAny>, target_config: &Bound<'py, PyAny>, cli_vars: Option>, @@ -159,126 +63,3 @@ fn discovered_vars<'py>( &cli_vars, ) } - -/// Build effective target config without compiling resources. -#[pyfunction] -#[pyo3(signature = (*, discovered_inputs, selected_target = None))] -pub(crate) fn build_effective_target_config<'py>( - discovered_inputs: &Bound<'py, PyAny>, - selected_target: Option>, -) -> PyResult> { - Ok(target_selection(discovered_inputs, selected_target)?.1) -} - -/// Build the effective project connection config without compiling resources. -#[pyfunction] -#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None, catalog = false))] -pub(crate) fn build_effective_connection_config<'py>( - discovered_inputs: &Bound<'py, PyAny>, - selected_target: Option>, - cli_vars: Option>, - catalog: bool, -) -> PyResult> { - let (_, target_config) = target_selection(discovered_inputs, selected_target)?; - let effective_vars: Bound<'py, PyAny> = - discovered_vars(discovered_inputs, &target_config, cli_vars)?; - expanded_connection( - ( - &native_field(discovered_inputs, "project_config")?, - &native_field(discovered_inputs, "local_config")?, - ), - &target_config, - &effective_vars, - catalog, - ) -} - -/// Build environment, vars and invocation id without compiling resources. -#[pyfunction] -#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None))] -pub(crate) fn build_effective_runtime_config<'py>( - discovered_inputs: &Bound<'py, PyAny>, - selected_target: Option>, - cli_vars: Option>, -) -> PyResult> { - let py: Python<'py> = discovered_inputs.py(); - let (target_name, target_config) = target_selection(discovered_inputs, selected_target)?; - let effective_vars: Bound<'py, PyAny> = - discovered_vars(discovered_inputs, &target_config, cli_vars)?; - let run_id: Bound<'py, PyAny> = run_id(&py.None().into_bound(py))?; - PyTuple::new(py, [target_name, effective_vars, run_id]) -} - -/// Build effective settings without compiling resources. -#[pyfunction] -#[pyo3(signature = (*, discovered_inputs))] -pub(crate) fn build_effective_settings_config<'py>( - discovered_inputs: &Bound<'py, PyAny>, -) -> PyResult> { - build_effective_settings( - &native_field(discovered_inputs, "project_config")?, - &native_field(discovered_inputs, "local_config")?, - ) -} - -/// `build_effective_vars` for a discovered project and target, without CLI vars. -pub(crate) fn project_vars<'py>( - discovered: &Bound<'py, PyAny>, - target_config: &Bound<'py, PyAny>, -) -> PyResult> { - build_effective_vars( - &native_field(discovered, "project_config")?, - &native_field(discovered, "local_config")?, - target_config, - PyDict::new(discovered.py()).as_any(), - ) -} - -/// `build_effective_connection` as a function of `module`. -pub(crate) fn build_effective_connection_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_effective_connection, module) -} - -/// `build_effective_connection_config` as a function of `module`. -pub(crate) fn build_effective_connection_config_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_effective_connection_config, module) -} - -/// `build_effective_runtime_config` as a function of `module`. -pub(crate) fn build_effective_runtime_config_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_effective_runtime_config, module) -} - -/// `build_effective_settings` as a function of `module`. -pub(crate) fn build_effective_settings_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_effective_settings, module) -} - -/// `build_effective_settings_config` as a function of `module`. -pub(crate) fn build_effective_settings_config_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_effective_settings_config, module) -} - -/// `build_effective_target_config` as a function of `module`. -pub(crate) fn build_effective_target_config_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_effective_target_config, module) -} - -/// `build_effective_vars` as a function of `module`. -pub(crate) fn build_effective_vars_function<'py>( - module: &pyo3::Bound<'py, pyo3::types::PyModule>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(build_effective_vars, module) -} diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection.rs index 1e07462d8f..8829856f5c 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection.rs @@ -1,22 +1,22 @@ //! Merge base project connection with the selected environment overrides. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compile_attachment::_helpers::effective_config::expanded_connection; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// Merge base project connection with the selected environment overrides. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, target_config, effective_vars))] pub fn build_effective_connection<'py>( project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, target_config: &Bound<'py, PyAny>, effective_vars: &Bound<'py, PyAny>, ) -> PyResult> { - effective_config::build_effective_connection( - project_config, - local_config, + expanded_connection( + (project_config, local_config), target_config, effective_vars, + false, ) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config.rs index 6bde72927b..163d413662 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config.rs @@ -1,22 +1,31 @@ //! Build the effective project connection config without compiling resources. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compile_attachment::_helpers::effective_config::discovered_vars; +use crate::compile_attachment::_helpers::effective_config::expanded_connection; +use crate::compile_attachment::main::target_selection::target_selection; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Build the effective project connection config without compiling resources. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None, catalog = false))] pub fn build_effective_connection_config<'py>( discovered_inputs: &Bound<'py, PyAny>, selected_target: Option>, cli_vars: Option>, catalog: bool, ) -> PyResult> { - effective_config::build_effective_connection_config( - discovered_inputs, - selected_target, - cli_vars, + let (_, target_config) = target_selection(discovered_inputs, selected_target)?; + let effective_vars: Bound<'py, PyAny> = + discovered_vars(discovered_inputs, &target_config, cli_vars)?; + expanded_connection( + ( + &native_field(discovered_inputs, "project_config")?, + &native_field(discovered_inputs, "local_config")?, + ), + &target_config, + &effective_vars, catalog, ) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config_function.rs index 74e098cc13..baa40b6a27 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config_function.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config_function.rs @@ -1,11 +1,10 @@ //! `build_effective_connection_config` as a function of `module`. -use crate::compile_attachment::_helpers::effective_config; +use crate::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; /// `build_effective_connection_config` as a function of `module`. -#[inline] pub fn build_effective_connection_config_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - effective_config::build_effective_connection_config_function(module) + pyo3::wrap_pyfunction!(build_effective_connection_config, module) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_function.rs index 1fd50f688d..7bc55e4d04 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_function.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_function.rs @@ -1,11 +1,10 @@ //! `build_effective_connection` as a function of `module`. -use crate::compile_attachment::_helpers::effective_config; +use crate::compile_attachment::main::build_effective_connection::build_effective_connection; /// `build_effective_connection` as a function of `module`. -#[inline] pub fn build_effective_connection_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - effective_config::build_effective_connection_function(module) + pyo3::wrap_pyfunction!(build_effective_connection, module) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config.rs index c4eeb120e1..a9d2c10702 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config.rs @@ -1,17 +1,24 @@ //! Build environment, vars and invocation id without compiling resources. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compile_attachment::_helpers::effective_config::discovered_vars; +use crate::compile_attachment::main::target_selection::target_selection; +use crate::model_config::main::config::run_id::run_id; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyTuple; /// Build environment, vars and invocation id without compiling resources. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target = None, cli_vars = None))] pub fn build_effective_runtime_config<'py>( discovered_inputs: &Bound<'py, PyAny>, selected_target: Option>, cli_vars: Option>, ) -> PyResult> { - effective_config::build_effective_runtime_config(discovered_inputs, selected_target, cli_vars) + let py: Python<'py> = discovered_inputs.py(); + let (target_name, target_config) = target_selection(discovered_inputs, selected_target)?; + let effective_vars: Bound<'py, PyAny> = + discovered_vars(discovered_inputs, &target_config, cli_vars)?; + let run_id: Bound<'py, PyAny> = run_id(&py.None().into_bound(py))?; + PyTuple::new(py, [target_name, effective_vars, run_id]) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config_function.rs index 2626522098..2ce5569a41 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config_function.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config_function.rs @@ -1,11 +1,10 @@ //! `build_effective_runtime_config` as a function of `module`. -use crate::compile_attachment::_helpers::effective_config; +use crate::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; /// `build_effective_runtime_config` as a function of `module`. -#[inline] pub fn build_effective_runtime_config_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - effective_config::build_effective_runtime_config_function(module) + pyo3::wrap_pyfunction!(build_effective_runtime_config, module) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings.rs index d5b3f04d24..beaf407f2e 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings.rs @@ -1,15 +1,42 @@ //! Merge project settings with local developer overrides. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::contracts::main::effective_config_settings::effective_config_settings; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyString}; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// Merge project settings with local developer overrides. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config))] pub fn build_effective_settings<'py>( project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, ) -> PyResult> { - effective_config::build_effective_settings(project_config, local_config) + let py: Python<'py> = project_config.py(); + let settings_class: Bound<'py, PyAny> = record_class(py, RecordId::SettingsConfig)?; + let names: Vec> = imported(py, "dataclasses", "fields")? + .call1((&settings_class,))? + .try_iter()? + .map(|field| native_field(&field?, "name")) + .collect::>()?; + let values: Bound<'py, PyDict> = effective_config_settings( + py, + ( + native_field(project_config, "settings")?, + native_field(local_config, "settings")?, + ), + &native_field(local_config, "setting_overrides")?, + names + .into_iter() + .map(|name| Ok(name.cast_into::()?)) + .collect::>()?, + ) + .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? + .downcast_into()?; + new_record(py, RecordId::SettingsConfig, (), Some(&values)) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config.rs index 7212e9a1bc..95f9b3632f 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config.rs @@ -1,14 +1,18 @@ //! Build effective settings without compiling resources. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compile_attachment::main::build_effective_settings::build_effective_settings; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Build effective settings without compiling resources. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs))] pub fn build_effective_settings_config<'py>( discovered_inputs: &Bound<'py, PyAny>, ) -> PyResult> { - effective_config::build_effective_settings_config(discovered_inputs) + build_effective_settings( + &native_field(discovered_inputs, "project_config")?, + &native_field(discovered_inputs, "local_config")?, + ) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config_function.rs index 2bc134c68c..ffc2ba7d9d 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config_function.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_config_function.rs @@ -1,11 +1,10 @@ //! `build_effective_settings_config` as a function of `module`. -use crate::compile_attachment::_helpers::effective_config; +use crate::compile_attachment::main::build_effective_settings_config::build_effective_settings_config; /// `build_effective_settings_config` as a function of `module`. -#[inline] pub fn build_effective_settings_config_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - effective_config::build_effective_settings_config_function(module) + pyo3::wrap_pyfunction!(build_effective_settings_config, module) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_function.rs index 4031736371..5699648601 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_function.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings_function.rs @@ -1,11 +1,10 @@ //! `build_effective_settings` as a function of `module`. -use crate::compile_attachment::_helpers::effective_config; +use crate::compile_attachment::main::build_effective_settings::build_effective_settings; /// `build_effective_settings` as a function of `module`. -#[inline] pub fn build_effective_settings_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - effective_config::build_effective_settings_function(module) + pyo3::wrap_pyfunction!(build_effective_settings, module) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config.rs index e7f58c2b4e..b6176d69e9 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config.rs @@ -1,15 +1,15 @@ //! Build effective target config without compiling resources. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compile_attachment::main::target_selection::target_selection; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// Build effective target config without compiling resources. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target = None))] pub fn build_effective_target_config<'py>( discovered_inputs: &Bound<'py, PyAny>, selected_target: Option>, ) -> PyResult> { - effective_config::build_effective_target_config(discovered_inputs, selected_target) + Ok(target_selection(discovered_inputs, selected_target)?.1) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config_function.rs index c15c82fa8c..1b53f61b51 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config_function.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_target_config_function.rs @@ -1,11 +1,10 @@ //! `build_effective_target_config` as a function of `module`. -use crate::compile_attachment::_helpers::effective_config; +use crate::compile_attachment::main::build_effective_target_config::build_effective_target_config; /// `build_effective_target_config` as a function of `module`. -#[inline] pub fn build_effective_target_config_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - effective_config::build_effective_target_config_function(module) + pyo3::wrap_pyfunction!(build_effective_target_config, module) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars.rs index 5225b0caad..f4b0350869 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars.rs @@ -1,17 +1,26 @@ //! Merge effective vars using the locked precedence order. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::contracts::main::effective_config_vars::effective_config_vars; +use crate::model_config::main::config::expanded_effective_vars::expanded_effective_vars; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; /// Merge effective vars using the locked precedence order. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, target_config, cli_vars))] pub fn build_effective_vars<'py>( project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, target_config: &Bound<'py, PyAny>, cli_vars: &Bound<'py, PyAny>, ) -> PyResult> { - effective_config::build_effective_vars(project_config, local_config, target_config, cli_vars) + let py: Python<'py> = project_config.py(); + let merged: Bound<'py, PyAny> = effective_config_vars( + py, + (project_config.clone(), local_config.clone()), + target_config, + cli_vars, + ) + .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; + expanded_effective_vars(merged.downcast()?) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars_function.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars_function.rs index fa6d8a7d07..d566e756f8 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars_function.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_vars_function.rs @@ -1,11 +1,10 @@ //! `build_effective_vars` as a function of `module`. -use crate::compile_attachment::_helpers::effective_config; +use crate::compile_attachment::main::build_effective_vars::build_effective_vars; /// `build_effective_vars` as a function of `module`. -#[inline] pub fn build_effective_vars_function<'py>( module: &pyo3::Bound<'py, pyo3::types::PyModule>, ) -> pyo3::PyResult> { - effective_config::build_effective_vars_function(module) + pyo3::wrap_pyfunction!(build_effective_vars, module) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/project_vars.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/project_vars.rs index 3215cf81e9..70c5470939 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/project_vars.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/project_vars.rs @@ -1,15 +1,19 @@ //! `build_effective_vars` for a discovered project and target, without CLI vars. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::compile_attachment::main::build_effective_vars::build_effective_vars; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyDict; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `build_effective_vars` for a discovered project and target, without CLI vars. -#[inline] pub fn project_vars<'py>( discovered: &Bound<'py, PyAny>, target_config: &Bound<'py, PyAny>, ) -> PyResult> { - effective_config::project_vars(discovered, target_config) + build_effective_vars( + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, + target_config, + PyDict::new(discovered.py()).as_any(), + ) } diff --git a/crates/sqlbuild-python-front/src/compile_attachment/main/target_selection.rs b/crates/sqlbuild-python-front/src/compile_attachment/main/target_selection.rs index db99738dd0..a84523391c 100644 --- a/crates/sqlbuild-python-front/src/compile_attachment/main/target_selection.rs +++ b/crates/sqlbuild-python-front/src/compile_attachment/main/target_selection.rs @@ -1,15 +1,24 @@ //! The selected target name of a discovered project and its effective config, if any. -use crate::compile_attachment::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::contracts::main::resolve_target_config::resolve_target_config; +use crate::contracts::main::resolve_target_name::resolve_target_name; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The selected target name of a discovered project and its effective config, if any. -#[inline] pub fn target_selection<'py>( discovered: &Bound<'py, PyAny>, selected: Option>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - effective_config::target_selection(discovered, selected) + let py: Python<'py> = discovered.py(); + let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; + let selected: Bound<'py, PyAny> = selected.unwrap_or_else(|| py.None().into_bound(py)); + let Some(name) = resolve_target_name(py, &project_config, &local_config, &selected)? else { + return Ok((py.None().into_bound(py), py.None().into_bound(py))); + }; + let name: Bound<'py, PyAny> = name.into_any(); + let config: Bound<'py, PyAny> = + resolve_target_config(py, &project_config, &local_config, &name)?; + Ok((name, config)) } diff --git a/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs b/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs index 3cc8fdb394..24d8b4bfd4 100644 --- a/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs +++ b/crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs @@ -1,21 +1,17 @@ //! Validate loaded project and local configuration before discovery reads project files. +use crate::compile_driver::main::session_warehouse_support::session_warehouse_support; use crate::compile_driver::models::ProjectConfiguration; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyString, PyTuple}; use sqlbuild_config::project::main::unsupported_target_warehouse::unsupported_target_warehouse; -use sqlbuild_host::entries::models::AdapterHost; - -use sqlbuild_python_base::boundary::main::call_with::call_with; - use crate::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use crate::contracts::main::resolve_target_config::resolve_target_config; use crate::discovery::main::project::resolve_config_file_path::resolve_config_file_path; use crate::discovery::main::project::validate_local_sql_analysis::validate_local_sql_analysis; use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; @@ -45,7 +41,9 @@ fn target_names(config: &Bound<'_, PyAny>) -> PyResult> { } /// Reject a local SQL analysis opt-out the project forbids. -fn reject_local_sql_analysis_opt_out(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { +pub(crate) fn reject_local_sql_analysis_opt_out( + config: &ProjectConfiguration<'_, '_>, +) -> PyResult<()> { let py: Python<'_> = config.project_dir.py(); let local_settings: Bound<'_, PyAny> = native_field(config.local_config, "settings")?; let flags: Bound<'_, PyTuple> = PyTuple::new( @@ -78,22 +76,6 @@ fn first_configured_group(config: &Bound<'_, PyAny>) -> PyResult, - adapter_name: &Bound<'_, PyAny>, - project_dir: &Bound<'_, PyAny>, -) -> PyResult { - call_with( - &host_entry(py, AdapterHost::AdapterSupportsSessionWarehouse)?, - &[ - ("adapter_name", adapter_name.clone()), - ("project_dir", project_dir.clone()), - ], - )? - .is_truthy() -} - fn reject_group( py: Python<'_>, (local, adapter_name): (bool, &Bound<'_, PyAny>), @@ -120,7 +102,9 @@ fn reject_group( } /// Reject command-group warehouses under adapters without a session warehouse. -fn reject_unsupported_target_warehouses(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { +pub(crate) fn reject_unsupported_target_warehouses( + config: &ProjectConfiguration<'_, '_>, +) -> PyResult<()> { let py: Python<'_> = config.project_dir.py(); if let Some(group) = first_configured_group(config.project_config)? { let adapter_name: Bound<'_, PyAny> = native_field(config.project_config, "adapter")?; @@ -153,7 +137,9 @@ fn reject_unsupported_target_warehouses(config: &ProjectConfiguration<'_, '_>) - } /// Resolve every configured target so invalid connections fail discovery. -fn reject_invalid_target_connections(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { +pub(crate) fn reject_invalid_target_connections( + config: &ProjectConfiguration<'_, '_>, +) -> PyResult<()> { let py: Python<'_> = config.project_dir.py(); let mut names: Vec = target_names(config.project_config)?; names.extend(target_names(config.local_config)?); @@ -181,19 +167,3 @@ fn reject_invalid_target_connections(config: &ProjectConfiguration<'_, '_>) -> P } Ok(()) } - -/// Validate the SQL analysis policy, target warehouses and target connections, in that order. -pub(crate) fn reject_invalid_project_configuration( - config: &ProjectConfiguration<'_, '_>, -) -> PyResult<()> { - reject_local_sql_analysis_opt_out(config)?; - reject_invalid_connection_configuration(config) -} - -/// Validate the target warehouses and target connections a warehouse connection depends on. -pub(crate) fn reject_invalid_connection_configuration( - config: &ProjectConfiguration<'_, '_>, -) -> PyResult<()> { - reject_unsupported_target_warehouses(config)?; - reject_invalid_target_connections(config) -} diff --git a/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_connection_configuration.rs b/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_connection_configuration.rs index 4dab78eccb..17960d30ae 100644 --- a/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_connection_configuration.rs +++ b/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_connection_configuration.rs @@ -1,13 +1,14 @@ //! Validate the target warehouses and target connections a warehouse connection depends on. -use crate::compile_driver::_helpers::config_validation; +use crate::compile_driver::_helpers::config_validation::reject_invalid_target_connections; +use crate::compile_driver::_helpers::config_validation::reject_unsupported_target_warehouses; use crate::compile_driver::models::ProjectConfiguration; use pyo3::prelude::PyResult; /// Validate the target warehouses and target connections a warehouse connection depends on. -#[inline] pub fn reject_invalid_connection_configuration( config: &ProjectConfiguration<'_, '_>, ) -> PyResult<()> { - config_validation::reject_invalid_connection_configuration(config) + reject_unsupported_target_warehouses(config)?; + reject_invalid_target_connections(config) } diff --git a/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_project_configuration.rs b/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_project_configuration.rs index f7afdf5744..c9523744fa 100644 --- a/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_project_configuration.rs +++ b/crates/sqlbuild-python-front/src/compile_driver/main/reject_invalid_project_configuration.rs @@ -1,11 +1,12 @@ //! Validate the SQL analysis policy, target warehouses and target connections, in that order. -use crate::compile_driver::_helpers::config_validation; +use crate::compile_driver::_helpers::config_validation::reject_local_sql_analysis_opt_out; +use crate::compile_driver::main::reject_invalid_connection_configuration::reject_invalid_connection_configuration; use crate::compile_driver::models::ProjectConfiguration; use pyo3::prelude::PyResult; /// Validate the SQL analysis policy, target warehouses and target connections, in that order. -#[inline] pub fn reject_invalid_project_configuration(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { - config_validation::reject_invalid_project_configuration(config) + reject_local_sql_analysis_opt_out(config)?; + reject_invalid_connection_configuration(config) } diff --git a/crates/sqlbuild-python-front/src/compile_driver/main/session_warehouse_support.rs b/crates/sqlbuild-python-front/src/compile_driver/main/session_warehouse_support.rs index 72ca9d4c1e..c5e3947899 100644 --- a/crates/sqlbuild-python-front/src/compile_driver/main/session_warehouse_support.rs +++ b/crates/sqlbuild-python-front/src/compile_driver/main/session_warehouse_support.rs @@ -1,17 +1,22 @@ //! Whether the adapter can switch warehouses per session, as the adapter host answers. -use crate::compile_driver::_helpers::config_validation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; /// Whether the adapter can switch warehouses per session, as the adapter host answers. -#[inline] pub fn session_warehouse_support( py: Python<'_>, adapter_name: &Bound<'_, PyAny>, project_dir: &Bound<'_, PyAny>, ) -> PyResult { - config_validation::session_warehouse_support(py, adapter_name, project_dir) + call_with( + &host_entry(py, AdapterHost::AdapterSupportsSessionWarehouse)?, + &[ + ("adapter_name", adapter_name.clone()), + ("project_dir", project_dir.clone()), + ], + )? + .is_truthy() } diff --git a/crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs b/crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs index 4539fa9c21..ad700a9f8a 100644 --- a/crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs +++ b/crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs @@ -1,23 +1,18 @@ //! Effective target, vars, connection and settings precedence over the loaded config objects. +use crate::contracts::main::effective_connection_fields::effective_connection_fields; use std::hash::{Hash, Hasher}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, PyStringMethods, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, PyStringMethods, Python}; +use pyo3::pyfunction; use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyListMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_config::effective::errors::{EffectiveConfigError, MessagePart}; use sqlbuild_config::effective::main::default_connection_name::default_connection_name; -use sqlbuild_config::effective::main::effective_connection::effective_connection; -use sqlbuild_config::effective::main::effective_settings::effective_settings; use sqlbuild_config::effective::main::effective_target_name::effective_target_name; -use sqlbuild_config::effective::main::effective_vars::effective_vars; use sqlbuild_config::effective::main::merged_target::merged_target; use sqlbuild_config::effective::main::target_fields::target_fields; use sqlbuild_config::effective::models::{ - ConnectionChoice, ConnectionLayers, FieldValue, MergeRule, TargetChoice, TargetField, - TargetLayer, + ConnectionChoice, FieldValue, MergeRule, TargetChoice, TargetField, TargetLayer, }; use sqlbuild_config::effective::types::Entries; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -27,7 +22,7 @@ const SURROGATE_PASS: &str = "surrogatepass"; /// A `str` key compared exactly: by its UTF-8 bytes, lone surrogates passed through. #[derive(Clone)] -struct Key<'py> { +pub(crate) struct Key<'py> { bytes: Vec, handle: Bound<'py, PyString>, } @@ -58,15 +53,15 @@ impl Ord for Key<'_> { } } -type Value<'py> = Bound<'py, PyAny>; +pub(crate) type Value<'py> = Bound<'py, PyAny>; type Layer<'py> = TargetLayer, Value<'py>>; -type Pairs<'py> = Entries, Value<'py>>; +pub(crate) type Pairs<'py> = Entries, Value<'py>>; /// `(group, name, value)` of each merged `TargetConfig` field. type FieldRow<'py> = (Option<&'static str>, &'static str, Value<'py>); /// A result, else the precedence failure message. pub(crate) type Outcome<'py, T> = (Option, Option>); -fn key<'py>(value: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn key<'py>(value: &Bound<'py, PyAny>) -> PyResult> { let handle: Bound<'py, PyString> = value.downcast::()?.clone(); if let Ok(text) = handle.to_str() { return Ok(Key { @@ -82,14 +77,14 @@ fn key<'py>(value: &Bound<'py, PyAny>) -> PyResult> { Ok(Key { bytes, handle }) } -fn optional_key<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn optional_key<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { if value.is_none() { return Ok(None); } key(value).map(Some) } -fn pairs<'py>(mapping: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn pairs<'py>(mapping: &Bound<'py, PyAny>) -> PyResult> { mapping .downcast::()? .iter() @@ -106,7 +101,10 @@ fn keys<'py>(mapping: &Bound<'py, PyAny>) -> PyResult>> { .collect() } -fn dictionary<'py>(py: Python<'py>, entries: Pairs<'py>) -> PyResult> { +pub(crate) fn dictionary<'py>( + py: Python<'py>, + entries: Pairs<'py>, +) -> PyResult> { let dictionary: Bound<'py, PyDict> = PyDict::new(py); for (name, value) in entries { dictionary.set_item(name.handle, value)?; @@ -162,7 +160,7 @@ fn is_set(value: &FieldValue, Value<'_>>) -> bool { } /// The failure message as a Python `str`, authored names inserted unchanged. -fn message<'py>( +pub(crate) fn message<'py>( py: Python<'py>, error: &EffectiveConfigError>, ) -> PyResult> { @@ -251,27 +249,7 @@ pub(crate) fn effective_config_target_fields<'py>( Ok((Some(rows), None)) } -/// `build_effective_vars` before template expansion. -#[pyfunction] -pub(crate) fn effective_config_vars<'py>( - py: Python<'py>, - configs: (Bound<'py, PyAny>, Bound<'py, PyAny>), - target_config: &Bound<'py, PyAny>, - cli_vars: &Bound<'py, PyAny>, -) -> PyResult> { - let (project_config, local_config) = configs; - let project: Pairs<'py> = pairs(&native_field(&project_config, "vars")?)?; - let target: Pairs<'py> = if target_config.is_none() { - Vec::new() - } else { - pairs(&native_field(target_config, "vars")?)? - }; - let local: Pairs<'py> = pairs(&native_field(&local_config, "vars")?)?; - let cli: Pairs<'py> = pairs(cli_vars)?; - dictionary(py, effective_vars(&[&project, &target, &local, &cli])) -} - -fn named<'py>(mapping: &Bound<'py, PyAny>) -> PyResult, Pairs<'py>>> { +pub(crate) fn named<'py>(mapping: &Bound<'py, PyAny>) -> PyResult, Pairs<'py>>> { mapping .downcast::()? .iter() @@ -289,98 +267,3 @@ pub(crate) fn effective_config_connection<'py>( ) -> PyResult>> { effective_connection_fields(py, (project_config, local_config), target_config, false) } - -/// Catalog selection uses the same named-layer resolver, with no writer layers. -pub(crate) fn effective_connection_fields<'py>( - py: Python<'py>, - (project_config, local_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - target_config: &Bound<'py, PyAny>, - catalog: bool, -) -> PyResult>> { - let catalog_name = if catalog && !target_config.is_none() { - optional_key(&native_field(target_config, "catalog_connection")?)? - } else { - None - }; - let independent = catalog_name.is_some(); - let target: Option<(Option>, Pairs<'py>)> = if independent { - Some((catalog_name, Vec::new())) - } else if target_config.is_none() { - None - } else { - Some(( - optional_key(&native_field(target_config, "connection_name")?)?, - pairs(&native_field(target_config, "connection")?)?, - )) - }; - let layers: ConnectionLayers, Value<'py>> = ConnectionLayers { - project: if independent { - Vec::new() - } else { - pairs(&native_field(project_config, "connection")?)? - }, - target, - project_named: named(&native_field(project_config, "connections")?)?, - local_named: named(&native_field(local_config, "connections")?)?, - local: if independent { - Vec::new() - } else { - pairs(&native_field(local_config, "connection")?)? - }, - }; - Ok(match effective_connection(layers) { - Ok(connection) => (Some(dictionary(py, connection)?), None), - Err(error) => { - let message = message(py, &error)?; - ( - None, - Some(if independent { - PyString::new(py, &format!("catalog_connection: {message}")) - } else { - message - }), - ) - } - }) -} - -/// `build_effective_settings`: setting values by field name, local overrides applied. -#[pyfunction] -pub(crate) fn effective_config_settings<'py>( - py: Python<'py>, - settings: (Bound<'py, PyAny>, Bound<'py, PyAny>), - overrides: &Bound<'py, PyAny>, - names: Vec>, -) -> PyResult> { - let read = |settings: &Bound<'py, PyAny>| -> PyResult> { - names - .iter() - .map(|name| { - Ok(( - key(name.as_any())?, - native_field(settings, &name.to_cow()?)?, - )) - }) - .collect() - }; - let overridden: Vec> = overrides - .try_iter()? - .map(|name| key(&name?)) - .collect::>()?; - let (project_settings, local_settings) = settings; - let settings: Pairs<'py> = effective_settings( - read(&project_settings)?, - &read(&local_settings)?, - &overridden, - ); - dictionary(py, settings) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(effective_config_target_name, module)?)?; - module.add_function(wrap_pyfunction!(effective_config_target_fields, module)?)?; - module.add_function(wrap_pyfunction!(effective_config_vars, module)?)?; - module.add_function(wrap_pyfunction!(effective_config_connection, module)?)?; - module.add_function(wrap_pyfunction!(effective_config_settings, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs b/crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs index 6fd358ee29..a989a2f477 100644 --- a/crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs +++ b/crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs @@ -4,206 +4,57 @@ use std::cell::RefCell; use std::collections::HashMap; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyString}; use sqlbuild_model_config::name_patterns::main::compile_name_pattern::compile_name_pattern; use sqlbuild_model_config::name_patterns::models::NamePattern; -use crate::contracts::_helpers::effective_config::effective_config_target_fields; -use crate::contracts::_helpers::effective_config::effective_config_target_name; use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -const TARGET_GROUPS: [(&str, RecordId); 4] = [ +pub(crate) const TARGET_GROUPS: [(&str, RecordId); 4] = [ ("clone", RecordId::ClonePolicy), ("execution_limits", RecordId::ExecutionLimitsConfig), ("diff", RecordId::TargetDiffConfig), ("warehouses", RecordId::TargetWarehousesConfig), ]; -const SNAPSHOT_LIMITS: [&str; 4] = [ +pub(crate) const SNAPSHOT_LIMITS: [&str; 4] = [ "max_rows_per_relation", "max_total_rows", "max_bytes_per_relation", "max_total_bytes", ]; const ALL_DIALECTS: &str = "*"; -const LOADER_SCHEMA_TABLE_PARTS: usize = 2; -const LOADER_QUALIFIED_TABLE_PARTS: usize = 3; +pub(crate) const LOADER_SCHEMA_TABLE_PARTS: usize = 2; +pub(crate) const LOADER_QUALIFIED_TABLE_PARTS: usize = 3; const NAME_PATTERN_CACHE_LIMIT: usize = 256; thread_local! { static NAME_PATTERNS: RefCell> = RefCell::new(HashMap::new()); } -fn spec_error<'py>(py: Python<'py>, message: Bound<'py, PyString>) -> PyResult { +pub(crate) fn spec_error<'py>(py: Python<'py>, message: Bound<'py, PyString>) -> PyResult { Ok(PyErr::from_value( error_class(py, ErrorId::SpecConfig)?.call1((message,))?, )) } -/// Resolve the effective target name. -#[pyfunction] -#[pyo3(signature = (*, project_config, local_config, selected_target))] -pub(crate) fn resolve_target_name<'py>( - py: Python<'py>, - project_config: &Bound<'py, PyAny>, - local_config: &Bound<'py, PyAny>, - selected_target: &Bound<'py, PyAny>, -) -> PyResult>> { - let (name, error) = - effective_config_target_name(py, project_config, local_config, selected_target)?; - match error { - Some(message) => Err(spec_error(py, message)?), - None => Ok(name), - } -} - -/// Merge project target config with local developer overrides. -#[pyfunction] -#[pyo3(signature = (*, project_config, local_config, target_name))] -pub(crate) fn resolve_target_config<'py>( - py: Python<'py>, - project_config: &Bound<'py, PyAny>, - local_config: &Bound<'py, PyAny>, - target_name: &Bound<'py, PyAny>, -) -> PyResult> { - let (rows, error) = effective_config_target_fields( - py, - (project_config.clone(), local_config.clone()), - target_name, - &new_record(py, RecordId::TargetConfig, (), None)?, - )?; - let Some(rows) = rows else { - let message: Bound<'py, PyString> = error.unwrap_or_else(|| PyString::new(py, "")); - return Err(spec_error(py, message)?); - }; - let values: Bound<'py, PyDict> = PyDict::new(py); - let grouped: Vec<(&str, Bound<'py, PyDict>)> = TARGET_GROUPS - .iter() - .map(|(group, _)| (*group, PyDict::new(py))) - .collect(); - for (group, name, value) in rows { - match group { - None => values.set_item(name, value)?, - Some(group) => { - if let Some((_, fields)) = grouped.iter().find(|(known, _)| *known == group) { - fields.set_item(name, value)?; - } - } - } - } - for ((group, class_name), (_, fields)) in TARGET_GROUPS.iter().zip(&grouped) { - let nested: Bound<'py, PyAny> = new_record(py, *class_name, (), Some(fields))?; - values.set_item(*group, nested)?; - } - new_record(py, RecordId::TargetConfig, (), Some(&values)) -} - -/// Resolve the effective adapter name, allowing local override. -#[pyfunction] -#[pyo3(signature = (*, project_config, local_config))] -pub(crate) fn resolve_effective_adapter_name<'py>( - project_config: &Bound<'py, PyAny>, - local_config: &Bound<'py, PyAny>, -) -> PyResult> { - let local: Bound<'py, PyAny> = native_field(local_config, "adapter")?; - if local.is_none() { - return native_field(project_config, "adapter"); - } - Ok(local) -} - -/// Resolve declaration rendering over project configuration. -#[pyfunction] -#[pyo3(signature = (*, project_config, declaration_override))] -pub(crate) fn resolve_effective_collection_rendering<'py>( - project_config: &Bound<'py, PyAny>, - declaration_override: &Bound<'py, PyAny>, -) -> PyResult> { - if !declaration_override.is_none() { - return Ok(declaration_override.clone()); - } - native_field(project_config, "constants")?.getattr("collection_rendering") -} - /// `local` when set, else `project`. -fn override_or<'py>(local: Bound<'py, PyAny>, project: Bound<'py, PyAny>) -> Bound<'py, PyAny> { +pub(crate) fn override_or<'py>( + local: Bound<'py, PyAny>, + project: Bound<'py, PyAny>, +) -> Bound<'py, PyAny> { if local.is_none() { project } else { local } } -/// Resolve scenario config, allowing local overrides to replace project rules. -#[pyfunction] -#[pyo3(signature = (*, project_config, local_config))] -pub(crate) fn resolve_effective_scenario_config<'py>( - project_config: &Bound<'py, PyAny>, - local_config: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_config.py(); - let project: Bound<'py, PyAny> = native_field(project_config, "scenario")?; - let local: Bound<'py, PyAny> = native_field(local_config, "scenario")?; - let overrides: Bound<'py, PyDict> = PyDict::new(py); - for item in native_field(&project, "local_type_overrides")? - .call_method0("items")? - .try_iter()? - { - let (dialect, rules): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; - overrides.set_item(dialect, py.get_type::().call1((rules,))?)?; - } - for item in native_field(&local, "local_type_overrides")? - .call_method0("items")? - .try_iter()? - { - let (dialect, rules): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; - overrides - .call_method1("setdefault", (dialect, PyDict::new(py)))? - .call_method1("update", (rules,))?; - } - let project_limits: Bound<'py, PyAny> = native_field(&project, "snapshot_limits")?; - let local_limits: Bound<'py, PyAny> = native_field(&local, "snapshot_limits")?; - let limits: Vec<(&str, Bound<'py, PyAny>)> = SNAPSHOT_LIMITS - .iter() - .map(|field| { - Ok(( - *field, - override_or( - native_field(&local_limits, field)?, - native_field(&project_limits, field)?, - ), - )) - }) - .collect::>()?; - new_record_with( - py, - RecordId::ScenarioConfig, - &[ - ( - "run_namespace", - override_or( - native_field(&local, "run_namespace")?, - native_field(&project, "run_namespace")?, - ), - ), - ("local_type_overrides", overrides.into_any()), - ( - "snapshot_limits", - new_record_with(py, RecordId::ScenarioSnapshotLimitsConfig, &limits)?, - ), - ], - ) -} - /// Return global and dialect-specific scenario local type override rules. #[pyfunction] #[pyo3(signature = (*, scenario_config, sql_analysis_dialect))] -fn scenario_local_type_overrides_for_dialect<'py>( +pub(crate) fn scenario_local_type_overrides_for_dialect<'py>( scenario_config: &Bound<'py, PyAny>, sql_analysis_dialect: &Bound<'py, PyAny>, ) -> PyResult> { @@ -228,69 +79,13 @@ fn scenario_local_type_overrides_for_dialect<'py>( } /// Whether `text` is empty or only whitespace as Python's `str.strip` sees it. -fn blank(text: &str) -> bool { +pub(crate) fn blank(text: &str) -> bool { text.chars() .all(|character| character.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&character)) } -/// Parse a one-, two-, or three-part loader destination. -#[pyfunction] -#[pyo3(signature = (*, destination, default_database, default_schema))] -pub(crate) fn loader_destination_parts<'py>( - py: Python<'py>, - destination: &str, - default_database: Option>, - default_schema: Option>, -) -> PyResult> { - loader_parts(py, destination, (default_database, default_schema)) -} - -pub(crate) fn loader_parts<'py>( - py: Python<'py>, - destination: &str, - (default_database, default_schema): (Option>, Option>), -) -> PyResult> { - let parts: Vec<&str> = destination.split('.').collect(); - if !(1..=LOADER_QUALIFIED_TABLE_PARTS).contains(&parts.len()) - || parts.iter().any(|part| blank(part)) - { - return Err(spec_error( - py, - PyString::new( - py, - &format!( - "Invalid loader destination '{destination}': expected 1 to 3 non-empty \ - dot-separated parts (table, schema.table, or database.schema.table)" - ), - ), - )?); - } - let none = || py.None().into_bound(py); - let text = |part: &str| PyString::new(py, part).into_any(); - let (database, schema, table): (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>) = - match parts.as_slice() { - [table] => ( - default_database.unwrap_or_else(none), - default_schema.unwrap_or_else(none), - text(table), - ), - [schema, table] if parts.len() == LOADER_SCHEMA_TABLE_PARTS => ( - default_database.unwrap_or_else(none), - text(schema), - text(table), - ), - _ => (text(parts[0]), text(parts[1]), text(parts[2])), - }; - new_record( - py, - RecordId::LoaderDestinationParts, - (database, schema, table), - None, - ) -} - /// Whether a family's optional `name_pattern` accepts `column_name`. -fn family_accepts(family: &Bound<'_, PyAny>, column_name: &str) -> PyResult { +pub(crate) fn family_accepts(family: &Bound<'_, PyAny>, column_name: &str) -> PyResult { let pattern: Option = native_field(family, "name_pattern")?.extract()?; let Some(pattern) = pattern else { return Ok(true); @@ -310,38 +105,3 @@ fn family_accepts(family: &Bound<'_, PyAny>, column_name: &str) -> PyResult( - families: &Bound<'py, PyAny>, - column_name: &str, -) -> PyResult> { - let mut matching: Vec> = Vec::new(); - for family in families.try_iter()? { - let family: Bound<'py, PyAny> = family?; - if family_accepts(&family, column_name)? { - matching.push(family); - } - } - PyTuple::new(families.py(), matching) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_target_name, module)?)?; - module.add_function(wrap_pyfunction!(resolve_target_config, module)?)?; - module.add_function(wrap_pyfunction!(resolve_effective_adapter_name, module)?)?; - module.add_function(wrap_pyfunction!( - resolve_effective_collection_rendering, - module - )?)?; - module.add_function(wrap_pyfunction!(resolve_effective_scenario_config, module)?)?; - module.add_function(wrap_pyfunction!( - scenario_local_type_overrides_for_dialect, - module - )?)?; - module.add_function(wrap_pyfunction!(loader_destination_parts, module)?)?; - module.add_function(wrap_pyfunction!(matching_dynamic_families, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/contracts/main/effective_config_settings.rs b/crates/sqlbuild-python-front/src/contracts/main/effective_config_settings.rs index bfe3c29867..942f305cf2 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/effective_config_settings.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/effective_config_settings.rs @@ -1,20 +1,43 @@ //! `build_effective_settings`: setting values by field name, local overrides applied. -use crate::contracts::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; -use pyo3::types::PyString; +use crate::contracts::_helpers::effective_config::Key; +use crate::contracts::_helpers::effective_config::Pairs; +use crate::contracts::_helpers::effective_config::dictionary; +use crate::contracts::_helpers::effective_config::key; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, PyStringMethods, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyString}; +use sqlbuild_config::effective::main::effective_settings::effective_settings; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `build_effective_settings`: setting values by field name, local overrides applied. -#[inline] +#[pyfunction] pub fn effective_config_settings<'py>( py: Python<'py>, settings: (Bound<'py, PyAny>, Bound<'py, PyAny>), overrides: &Bound<'py, PyAny>, names: Vec>, ) -> PyResult> { - effective_config::effective_config_settings(py, settings, overrides, names) + let read = |settings: &Bound<'py, PyAny>| -> PyResult> { + names + .iter() + .map(|name| { + Ok(( + key(name.as_any())?, + native_field(settings, &name.to_cow()?)?, + )) + }) + .collect() + }; + let overridden: Vec> = overrides + .try_iter()? + .map(|name| key(&name?)) + .collect::>()?; + let (project_settings, local_settings) = settings; + let settings: Pairs<'py> = effective_settings( + read(&project_settings)?, + &read(&local_settings)?, + &overridden, + ); + dictionary(py, settings) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/effective_config_vars.rs b/crates/sqlbuild-python-front/src/contracts/main/effective_config_vars.rs index f1cad2a5b6..a9df435341 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/effective_config_vars.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/effective_config_vars.rs @@ -1,19 +1,30 @@ //! `build_effective_vars` before template expansion. -use crate::contracts::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::contracts::_helpers::effective_config::Pairs; +use crate::contracts::_helpers::effective_config::dictionary; +use crate::contracts::_helpers::effective_config::pairs; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_config::effective::main::effective_vars::effective_vars; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `build_effective_vars` before template expansion. -#[inline] +#[pyfunction] pub fn effective_config_vars<'py>( py: Python<'py>, configs: (Bound<'py, PyAny>, Bound<'py, PyAny>), target_config: &Bound<'py, PyAny>, cli_vars: &Bound<'py, PyAny>, ) -> PyResult> { - effective_config::effective_config_vars(py, configs, target_config, cli_vars) + let (project_config, local_config) = configs; + let project: Pairs<'py> = pairs(&native_field(&project_config, "vars")?)?; + let target: Pairs<'py> = if target_config.is_none() { + Vec::new() + } else { + pairs(&native_field(target_config, "vars")?)? + }; + let local: Pairs<'py> = pairs(&native_field(&local_config, "vars")?)?; + let cli: Pairs<'py> = pairs(cli_vars)?; + dictionary(py, effective_vars(&[&project, &target, &local, &cli])) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/effective_connection_fields.rs b/crates/sqlbuild-python-front/src/contracts/main/effective_connection_fields.rs index 24e86bbc01..715778081b 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/effective_connection_fields.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/effective_connection_fields.rs @@ -1,25 +1,70 @@ //! Catalog selection uses the same named-layer resolver, with no writer layers. -use crate::contracts::_helpers::effective_config; +use crate::contracts::_helpers::effective_config::Key; use crate::contracts::_helpers::effective_config::Outcome; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; +use crate::contracts::_helpers::effective_config::Pairs; +use crate::contracts::_helpers::effective_config::Value; +use crate::contracts::_helpers::effective_config::dictionary; +use crate::contracts::_helpers::effective_config::message; +use crate::contracts::_helpers::effective_config::named; +use crate::contracts::_helpers::effective_config::optional_key; +use crate::contracts::_helpers::effective_config::pairs; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString}; +use sqlbuild_config::effective::main::effective_connection::effective_connection; +use sqlbuild_config::effective::models::ConnectionLayers; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Catalog selection uses the same named-layer resolver, with no writer layers. -#[inline] pub fn effective_connection_fields<'py>( py: Python<'py>, (project_config, local_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), target_config: &Bound<'py, PyAny>, catalog: bool, ) -> PyResult>> { - effective_config::effective_connection_fields( - py, - (project_config, local_config), - target_config, - catalog, - ) + let catalog_name = if catalog && !target_config.is_none() { + optional_key(&native_field(target_config, "catalog_connection")?)? + } else { + None + }; + let independent = catalog_name.is_some(); + let target: Option<(Option>, Pairs<'py>)> = if independent { + Some((catalog_name, Vec::new())) + } else if target_config.is_none() { + None + } else { + Some(( + optional_key(&native_field(target_config, "connection_name")?)?, + pairs(&native_field(target_config, "connection")?)?, + )) + }; + let layers: ConnectionLayers, Value<'py>> = ConnectionLayers { + project: if independent { + Vec::new() + } else { + pairs(&native_field(project_config, "connection")?)? + }, + target, + project_named: named(&native_field(project_config, "connections")?)?, + local_named: named(&native_field(local_config, "connections")?)?, + local: if independent { + Vec::new() + } else { + pairs(&native_field(local_config, "connection")?)? + }, + }; + Ok(match effective_connection(layers) { + Ok(connection) => (Some(dictionary(py, connection)?), None), + Err(error) => { + let message = message(py, &error)?; + ( + None, + Some(if independent { + PyString::new(py, &format!("catalog_connection: {message}")) + } else { + message + }), + ) + } + }) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/loader_destination_parts.rs b/crates/sqlbuild-python-front/src/contracts/main/loader_destination_parts.rs index 77dd1720fb..29cd7607b7 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/loader_destination_parts.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/loader_destination_parts.rs @@ -1,18 +1,17 @@ //! Parse a one-, two-, or three-part loader destination. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::contracts::main::loader_parts::loader_parts; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; /// Parse a one-, two-, or three-part loader destination. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, destination, default_database, default_schema))] pub fn loader_destination_parts<'py>( py: Python<'py>, destination: &str, default_database: Option>, default_schema: Option>, ) -> PyResult> { - spec_resolution::loader_destination_parts(py, destination, default_database, default_schema) + loader_parts(py, destination, (default_database, default_schema)) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/loader_parts.rs b/crates/sqlbuild-python-front/src/contracts/main/loader_parts.rs index b710005f65..52d16022d7 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/loader_parts.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/loader_parts.rs @@ -1,17 +1,55 @@ //! Loader parts. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::contracts::_helpers::spec_resolution::LOADER_QUALIFIED_TABLE_PARTS; +use crate::contracts::_helpers::spec_resolution::LOADER_SCHEMA_TABLE_PARTS; +use crate::contracts::_helpers::spec_resolution::blank; +use crate::contracts::_helpers::spec_resolution::spec_error; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; /// Loader parts. -#[inline] pub fn loader_parts<'py>( py: Python<'py>, destination: &str, (default_database, default_schema): (Option>, Option>), ) -> PyResult> { - spec_resolution::loader_parts(py, destination, (default_database, default_schema)) + let parts: Vec<&str> = destination.split('.').collect(); + if !(1..=LOADER_QUALIFIED_TABLE_PARTS).contains(&parts.len()) + || parts.iter().any(|part| blank(part)) + { + return Err(spec_error( + py, + PyString::new( + py, + &format!( + "Invalid loader destination '{destination}': expected 1 to 3 non-empty \ + dot-separated parts (table, schema.table, or database.schema.table)" + ), + ), + )?); + } + let none = || py.None().into_bound(py); + let text = |part: &str| PyString::new(py, part).into_any(); + let (database, schema, table): (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>) = + match parts.as_slice() { + [table] => ( + default_database.unwrap_or_else(none), + default_schema.unwrap_or_else(none), + text(table), + ), + [schema, table] if parts.len() == LOADER_SCHEMA_TABLE_PARTS => ( + default_database.unwrap_or_else(none), + text(schema), + text(table), + ), + _ => (text(parts[0]), text(parts[1]), text(parts[2])), + }; + new_record( + py, + RecordId::LoaderDestinationParts, + (database, schema, table), + None, + ) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs b/crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs index 22ee8aafaf..93dc5b6819 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs @@ -1,16 +1,23 @@ //! Return declared dynamic families accepting a physical column name. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::contracts::_helpers::spec_resolution::family_accepts; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; /// Return declared dynamic families accepting a physical column name. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, families, column_name))] pub fn matching_dynamic_families<'py>( families: &Bound<'py, PyAny>, column_name: &str, ) -> PyResult> { - spec_resolution::matching_dynamic_families(families, column_name) + let mut matching: Vec> = Vec::new(); + for family in families.try_iter()? { + let family: Bound<'py, PyAny> = family?; + if family_accepts(&family, column_name)? { + matching.push(family); + } + } + PyTuple::new(families.py(), matching) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs b/crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs index 489cc16675..b0b7119ceb 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs @@ -1,12 +1,19 @@ //! Register the effective config functions and classes. -use crate::contracts::_helpers::effective_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::contracts::_helpers::effective_config::effective_config_connection; +use crate::contracts::_helpers::effective_config::effective_config_target_fields; +use crate::contracts::_helpers::effective_config::effective_config_target_name; +use crate::contracts::main::effective_config_settings::effective_config_settings; +use crate::contracts::main::effective_config_vars::effective_config_vars; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the effective config functions and classes. -#[inline] pub fn register_effective_config(module: &Bound<'_, PyModule>) -> PyResult<()> { - effective_config::register(module) + module.add_function(wrap_pyfunction!(effective_config_target_name, module)?)?; + module.add_function(wrap_pyfunction!(effective_config_target_fields, module)?)?; + module.add_function(wrap_pyfunction!(effective_config_vars, module)?)?; + module.add_function(wrap_pyfunction!(effective_config_connection, module)?)?; + module.add_function(wrap_pyfunction!(effective_config_settings, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/register_spec_resolution.rs b/crates/sqlbuild-python-front/src/contracts/main/register_spec_resolution.rs index f866a7667f..18e9f742cf 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/register_spec_resolution.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/register_spec_resolution.rs @@ -1,12 +1,31 @@ //! Register the spec resolution functions and classes. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::contracts::_helpers::spec_resolution::scenario_local_type_overrides_for_dialect; +use crate::contracts::main::loader_destination_parts::loader_destination_parts; +use crate::contracts::main::matching_dynamic_families::matching_dynamic_families; +use crate::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use crate::contracts::main::resolve_effective_collection_rendering::resolve_effective_collection_rendering; +use crate::contracts::main::resolve_effective_scenario_config::resolve_effective_scenario_config; +use crate::contracts::main::resolve_target_config::resolve_target_config; +use crate::contracts::main::resolve_target_name::resolve_target_name; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the spec resolution functions and classes. -#[inline] pub fn register_spec_resolution(module: &Bound<'_, PyModule>) -> PyResult<()> { - spec_resolution::register(module) + module.add_function(wrap_pyfunction!(resolve_target_name, module)?)?; + module.add_function(wrap_pyfunction!(resolve_target_config, module)?)?; + module.add_function(wrap_pyfunction!(resolve_effective_adapter_name, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_effective_collection_rendering, + module + )?)?; + module.add_function(wrap_pyfunction!(resolve_effective_scenario_config, module)?)?; + module.add_function(wrap_pyfunction!( + scenario_local_type_overrides_for_dialect, + module + )?)?; + module.add_function(wrap_pyfunction!(loader_destination_parts, module)?)?; + module.add_function(wrap_pyfunction!(matching_dynamic_families, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_adapter_name.rs b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_adapter_name.rs index 5da624dbdf..6888e0382c 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_adapter_name.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_adapter_name.rs @@ -1,15 +1,19 @@ //! Resolve the effective adapter name, allowing local override. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Resolve the effective adapter name, allowing local override. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config))] pub fn resolve_effective_adapter_name<'py>( project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, ) -> PyResult> { - spec_resolution::resolve_effective_adapter_name(project_config, local_config) + let local: Bound<'py, PyAny> = native_field(local_config, "adapter")?; + if local.is_none() { + return native_field(project_config, "adapter"); + } + Ok(local) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs index 9c29dfa1aa..ddff31f880 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs @@ -1,15 +1,18 @@ //! Resolve declaration rendering over project configuration. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Resolve declaration rendering over project configuration. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, declaration_override))] pub fn resolve_effective_collection_rendering<'py>( project_config: &Bound<'py, PyAny>, declaration_override: &Bound<'py, PyAny>, ) -> PyResult> { - spec_resolution::resolve_effective_collection_rendering(project_config, declaration_override) + if !declaration_override.is_none() { + return Ok(declaration_override.clone()); + } + native_field(project_config, "constants")?.getattr("collection_rendering") } diff --git a/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs index edce6d7b7b..0ea5527f76 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs @@ -1,15 +1,71 @@ //! Resolve scenario config, allowing local overrides to replace project rules. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::contracts::_helpers::spec_resolution::SNAPSHOT_LIMITS; +use crate::contracts::_helpers::spec_resolution::override_or; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// Resolve scenario config, allowing local overrides to replace project rules. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config))] pub fn resolve_effective_scenario_config<'py>( project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, ) -> PyResult> { - spec_resolution::resolve_effective_scenario_config(project_config, local_config) + let py: Python<'py> = project_config.py(); + let project: Bound<'py, PyAny> = native_field(project_config, "scenario")?; + let local: Bound<'py, PyAny> = native_field(local_config, "scenario")?; + let overrides: Bound<'py, PyDict> = PyDict::new(py); + for item in native_field(&project, "local_type_overrides")? + .call_method0("items")? + .try_iter()? + { + let (dialect, rules): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + overrides.set_item(dialect, py.get_type::().call1((rules,))?)?; + } + for item in native_field(&local, "local_type_overrides")? + .call_method0("items")? + .try_iter()? + { + let (dialect, rules): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + overrides + .call_method1("setdefault", (dialect, PyDict::new(py)))? + .call_method1("update", (rules,))?; + } + let project_limits: Bound<'py, PyAny> = native_field(&project, "snapshot_limits")?; + let local_limits: Bound<'py, PyAny> = native_field(&local, "snapshot_limits")?; + let limits: Vec<(&str, Bound<'py, PyAny>)> = SNAPSHOT_LIMITS + .iter() + .map(|field| { + Ok(( + *field, + override_or( + native_field(&local_limits, field)?, + native_field(&project_limits, field)?, + ), + )) + }) + .collect::>()?; + new_record_with( + py, + RecordId::ScenarioConfig, + &[ + ( + "run_namespace", + override_or( + native_field(&local, "run_namespace")?, + native_field(&project, "run_namespace")?, + ), + ), + ("local_type_overrides", overrides.into_any()), + ( + "snapshot_limits", + new_record_with(py, RecordId::ScenarioSnapshotLimitsConfig, &limits)?, + ), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/resolve_target_config.rs b/crates/sqlbuild-python-front/src/contracts/main/resolve_target_config.rs index 1b48b4863a..dc176befff 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/resolve_target_config.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/resolve_target_config.rs @@ -1,18 +1,51 @@ //! Merge project target config with local developer overrides. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::contracts::_helpers::effective_config::effective_config_target_fields; +use crate::contracts::_helpers::spec_resolution::TARGET_GROUPS; +use crate::contracts::_helpers::spec_resolution::spec_error; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; /// Merge project target config with local developer overrides. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, target_name))] pub fn resolve_target_config<'py>( py: Python<'py>, project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, target_name: &Bound<'py, PyAny>, ) -> PyResult> { - spec_resolution::resolve_target_config(py, project_config, local_config, target_name) + let (rows, error) = effective_config_target_fields( + py, + (project_config.clone(), local_config.clone()), + target_name, + &new_record(py, RecordId::TargetConfig, (), None)?, + )?; + let Some(rows) = rows else { + let message: Bound<'py, PyString> = error.unwrap_or_else(|| PyString::new(py, "")); + return Err(spec_error(py, message)?); + }; + let values: Bound<'py, PyDict> = PyDict::new(py); + let grouped: Vec<(&str, Bound<'py, PyDict>)> = TARGET_GROUPS + .iter() + .map(|(group, _)| (*group, PyDict::new(py))) + .collect(); + for (group, name, value) in rows { + match group { + None => values.set_item(name, value)?, + Some(group) => { + if let Some((_, fields)) = grouped.iter().find(|(known, _)| *known == group) { + fields.set_item(name, value)?; + } + } + } + } + for ((group, class_name), (_, fields)) in TARGET_GROUPS.iter().zip(&grouped) { + let nested: Bound<'py, PyAny> = new_record(py, *class_name, (), Some(fields))?; + values.set_item(*group, nested)?; + } + new_record(py, RecordId::TargetConfig, (), Some(&values)) } diff --git a/crates/sqlbuild-python-front/src/contracts/main/resolve_target_name.rs b/crates/sqlbuild-python-front/src/contracts/main/resolve_target_name.rs index ea408772b6..aa3d143d22 100644 --- a/crates/sqlbuild-python-front/src/contracts/main/resolve_target_name.rs +++ b/crates/sqlbuild-python-front/src/contracts/main/resolve_target_name.rs @@ -1,19 +1,24 @@ //! Resolve the effective target name. -use crate::contracts::_helpers::spec_resolution; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::contracts::_helpers::effective_config::effective_config_target_name; +use crate::contracts::_helpers::spec_resolution::spec_error; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyString; /// Resolve the effective target name. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, selected_target))] pub fn resolve_target_name<'py>( py: Python<'py>, project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, selected_target: &Bound<'py, PyAny>, ) -> PyResult>> { - spec_resolution::resolve_target_name(py, project_config, local_config, selected_target) + let (name, error) = + effective_config_target_name(py, project_config, local_config, selected_target)?; + match error { + Some(message) => Err(spec_error(py, message)?), + None => Ok(name), + } } diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/conversion.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/conversion.rs deleted file mode 100644 index 3e3dd1995e..0000000000 --- a/crates/sqlbuild-python-front/src/cursor/_helpers/conversion.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! One call's cursor values read from and built as the native cursor value records. - -use sqlbuild_python_base::native_host::main::host_error::host_error; - -use pyo3::prelude::{PyErr, Python}; -use sqlbuild_host::entries::models::FacadeHost; - -/// `CursorAlgebraError(message)`. -pub(crate) fn algebra_error(py: Python<'_>, message: &str) -> PyErr { - host_error(py, FacadeHost::CursorAlgebraError, message) -} diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs index 93150ef70a..a2994c6e93 100644 --- a/crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs @@ -1,50 +1,13 @@ //! Calendar durations parsed from text and applied to dates and timestamps. -use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyInt, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; -use sqlbuild_cursor::algebra::main::shift_by_duration::shift_by_duration; -use sqlbuild_cursor::algebra::models::{DurationDigits, DurationSpan, Moment}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_cursor::algebra::models::DurationSpan; -use crate::cursor::_helpers::values::civil_date; -use crate::cursor::_helpers::values::local_date_time; -use crate::cursor::_helpers::values::replaced; -use crate::cursor::constants::DATETIME_MODULE; -use crate::cursor::models::CursorSession; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; const MONTHS_PER_YEAR: i64 = 12; -/// `(years, months, days, hours, minutes, seconds)` of a duration like `1y6mo` or `30d`. -#[pyfunction] -#[pyo3(signature = (value))] -pub(crate) fn parse_cursor_duration<'py>( - py: Python<'py>, - value: &str, -) -> PyResult>> { - let Some(digits): Option = parse_duration(value) else { - return Ok(None); - }; - let int = py.get_type::(); - let parts: Vec> = [ - digits.years, - digits.months, - digits.days, - digits.hours, - digits.minutes, - digits.seconds, - ] - .iter() - .map(|text| int.call1((text.as_str(),))) - .collect::>()?; - PyTuple::new(py, parts).map(Some) -} - -fn duration_span(duration: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn duration_span(duration: &Bound<'_, PyAny>) -> PyResult { let field = |name: &str| -> PyResult { native_field(duration, name)?.extract() }; Ok(DurationSpan { months: field("years")? * MONTHS_PER_YEAR + field("months")?, @@ -54,51 +17,3 @@ fn duration_span(duration: &Bound<'_, PyAny>) -> PyResult { seconds: field("seconds")?, }) } - -/// `moment` (a `date` or `datetime`) moved by `span` in `direction`, keeping its type and `tzinfo`. -pub(crate) fn shifted_moment<'py>( - moment: &Bound<'py, PyAny>, - span: &DurationSpan, - direction: i64, -) -> PyResult> { - let py: Python<'py> = moment.py(); - let session: CursorSession<'py> = CursorSession::new(py)?; - let is_timestamp: bool = moment.is_instance(&imported(py, DATETIME_MODULE, "datetime")?)?; - let typed: Moment = if is_timestamp { - Moment::Timestamp { - local: local_date_time(moment)?, - fold: native_field(moment, "fold")?.extract::()? != 0, - } - } else { - Moment::Date(civil_date(moment)?) - }; - let shifted: Moment = session.checked(shift_by_duration(typed, span, direction))?; - match shifted { - Moment::Timestamp { local, fold } => replaced(moment, &local, fold), - Moment::Date(date) => call_with( - &native_field(moment, "replace")?, - &[ - ("year", date.year.into_pyobject(py)?.into_any()), - ("month", date.month.into_pyobject(py)?.into_any()), - ("day", date.day.into_pyobject(py)?.into_any()), - ], - ), - } -} - -/// `moment` moved forward (`direction` 1) or back (`direction` -1) by `duration`. -#[pyfunction] -#[pyo3(signature = (*, moment, duration, direction))] -pub(crate) fn shift_by_cursor_duration<'py>( - moment: &Bound<'py, PyAny>, - duration: &Bound<'py, PyAny>, - direction: i64, -) -> PyResult> { - shifted_moment(moment, &duration_span(duration)?, direction) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_cursor_duration, module)?)?; - module.add_function(wrap_pyfunction!(shift_by_cursor_duration, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs index a63c908d17..a709a0380b 100644 --- a/crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs @@ -1,24 +1,20 @@ //! Building, intersecting, merging, splitting and capping half-open cursor intervals. +use crate::cursor::main::scalars::as_scalar::as_scalar; use std::cmp::Ordering; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PySlice, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cursor::algebra::main::cap_from_end::cap_from_end; use sqlbuild_cursor::algebra::main::cap_from_start::cap_from_start; use sqlbuild_cursor::algebra::main::clamp::clamp; use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::main::merge::merge; -use sqlbuild_cursor::algebra::main::observed_partition::observed_partition; use sqlbuild_cursor::algebra::main::split_into_batches::split_into_batches; -use sqlbuild_cursor::algebra::main::validate_interval::validate_interval; use sqlbuild_cursor::algebra::models::{Interval, Scalar}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::cursor::_helpers::scalars::as_scalar; use crate::cursor::models::CursorSession; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -44,40 +40,10 @@ fn python_intervals<'py>( PyTuple::new(session.py(), built) } -/// Raise `CursorAlgebraError` unless the ends form a valid aligned interval. -#[pyfunction(name = "validate_cursor_interval")] -#[pyo3(signature = (*, start, end, grain))] -pub(crate) fn interval_validation( - start: &Bound<'_, PyAny>, - end: &Bound<'_, PyAny>, - grain: &Bound<'_, PyAny>, -) -> PyResult<()> { - let session: CursorSession<'_> = CursorSession::new(start.py())?; - let interval = Interval { - start: session.scalar(start)?, - end: session.scalar(end)?, - grain: session.grain(grain)?, - }; - session.checked(validate_interval(&interval, &session)) -} - -/// The canonical partition containing an observed physical value. -#[pyfunction(name = "cursor_observed_partition")] -#[pyo3(signature = (*, value, grain))] -pub(crate) fn partition<'py>( - value: &Bound<'py, PyAny>, - grain: &Bound<'py, PyAny>, -) -> PyResult> { - let session: CursorSession<'py> = CursorSession::new(value.py())?; - let scalar: Scalar = session.scalar(value)?; - let interval: Interval = session.checked(observed_partition(&scalar, session.grain(grain)?))?; - session.python_interval(&interval) -} - /// The intersection of two compatible half-open intervals, or `None`. #[pyfunction(name = "cursor_clamp")] #[pyo3(signature = (*, interval, bounds))] -fn clamp_intervals<'py>( +pub(crate) fn clamp_intervals<'py>( interval: &Bound<'py, PyAny>, bounds: &Bound<'py, PyAny>, ) -> PyResult>> { @@ -93,7 +59,7 @@ fn clamp_intervals<'py>( /// Overlapping or adjacent canonical intervals merged. #[pyfunction(name = "cursor_merge")] #[pyo3(signature = (*, intervals))] -fn merge_intervals<'py>(intervals: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn merge_intervals<'py>(intervals: &Bound<'py, PyAny>) -> PyResult> { let session: CursorSession<'py> = CursorSession::new(intervals.py())?; let typed: Vec = intervals .try_iter()? @@ -106,7 +72,7 @@ fn merge_intervals<'py>(intervals: &Bound<'py, PyAny>) -> PyResult( +pub(crate) fn merge_raw_intervals<'py>( intervals: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult> { @@ -167,7 +133,7 @@ fn merge_raw_intervals<'py>( /// An interval split into batches of a positive number of grain units. #[pyfunction(name = "cursor_split_into_batches")] #[pyo3(signature = (*, interval, step))] -fn split<'py>(interval: &Bound<'py, PyAny>, step: i64) -> PyResult> { +pub(crate) fn split<'py>(interval: &Bound<'py, PyAny>, step: i64) -> PyResult> { let session: CursorSession<'py> = CursorSession::new(interval.py())?; let typed: Interval = session.interval(interval)?; let batches: Vec = session.checked(split_into_batches(&typed, step, &session))?; @@ -185,28 +151,22 @@ fn sliced<'py>( /// At most `count` intervals from the start. #[pyfunction(name = "cursor_cap_from_start")] #[pyo3(signature = (*, intervals, count))] -fn cap_start<'py>(intervals: &Bound<'py, PyAny>, count: i64) -> PyResult> { +pub(crate) fn cap_start<'py>( + intervals: &Bound<'py, PyAny>, + count: i64, +) -> PyResult> { sliced(intervals, cap_from_start(intervals.len()?, count)) } /// At most `count` intervals from the end; an empty tuple when `count` is not positive. #[pyfunction(name = "cursor_cap_from_end")] #[pyo3(signature = (*, intervals, count))] -fn cap_end<'py>(intervals: &Bound<'py, PyAny>, count: i64) -> PyResult> { +pub(crate) fn cap_end<'py>( + intervals: &Bound<'py, PyAny>, + count: i64, +) -> PyResult> { if count <= 0 { return Ok(PyTuple::empty(intervals.py()).into_any()); } sliced(intervals, cap_from_end(intervals.len()?, count)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(interval_validation, module)?)?; - module.add_function(wrap_pyfunction!(partition, module)?)?; - module.add_function(wrap_pyfunction!(clamp_intervals, module)?)?; - module.add_function(wrap_pyfunction!(merge_intervals, module)?)?; - module.add_function(wrap_pyfunction!(merge_raw_intervals, module)?)?; - module.add_function(wrap_pyfunction!(split, module)?)?; - module.add_function(wrap_pyfunction!(cap_start, module)?)?; - module.add_function(wrap_pyfunction!(cap_end, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/mod.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/mod.rs index fffa7c18fe..c5e240fb0a 100644 --- a/crates/sqlbuild-python-front/src/cursor/_helpers/mod.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/mod.rs @@ -1,4 +1,3 @@ -pub(crate) mod conversion; pub(crate) mod durations; pub(crate) mod intervals; pub(crate) mod overrides; diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs index 05e7930c02..55b10209da 100644 --- a/crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs @@ -1,62 +1,14 @@ //! Executor cursor bounds from `--start-cursor`/`--end-cursor` overrides. -use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyInt, PyString, PyStringMethods}; -use sqlbuild_cursor::algebra::main::parse_integer_text::parse_integer_text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use crate::cursor::_helpers::conversion::algebra_error; -use crate::cursor::constants::DATETIME_MODULE; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -const DECIMAL_MODULE: &str = "decimal"; -const INTEGER_AS_TIMESTAMP: &str = "an integer cursor cannot be used as a timestamp"; -const TIMESTAMP_AS_INTEGER: &str = "a timestamp cursor cannot be used as an integer"; +pub(crate) const DECIMAL_MODULE: &str = "decimal"; +pub(crate) const INTEGER_AS_TIMESTAMP: &str = "an integer cursor cannot be used as a timestamp"; +pub(crate) const TIMESTAMP_AS_INTEGER: &str = "a timestamp cursor cannot be used as an integer"; -fn instance_of(value: &Bound<'_, PyAny>, class: RecordId) -> PyResult { +pub(crate) fn instance_of(value: &Bound<'_, PyAny>, class: RecordId) -> PyResult { value.is_instance(&record_class(value.py(), class)?) } - -/// A timestamp override as a `datetime`; dates start at midnight and text is ISO 8601. -pub(crate) fn cursor_timestamp<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = value.py(); - if value.is_none() { - return Ok(value.clone()); - } - if instance_of(value, RecordId::TimestampValue)? { - return native_field(value, "value"); - } - let datetime: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; - if instance_of(value, RecordId::DateValue)? { - let midnight: Bound<'py, PyAny> = native_field(&datetime, "min")?.call_method0("time")?; - return datetime.call_method1("combine", (native_field(value, "value")?, midnight)); - } - if instance_of(value, RecordId::IntegerValue)? { - return Err(algebra_error(py, INTEGER_AS_TIMESTAMP)); - } - datetime.call_method1("fromisoformat", (value,)) -} - -/// An integer override as an `int`; text is read the way `int(Decimal(text))` reads it. -pub(crate) fn cursor_integer<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = value.py(); - if value.is_none() { - return Ok(value.clone()); - } - if instance_of(value, RecordId::IntegerValue)? { - return native_field(value, "value"); - } - if instance_of(value, RecordId::DateValue)? || instance_of(value, RecordId::TimestampValue)? { - return Err(algebra_error(py, TIMESTAMP_AS_INTEGER)); - } - if let Ok(text) = value.downcast::() - && let Some(parsed) = parse_integer_text(text.to_str()?) - { - return Ok(parsed.into_pyobject(py)?.into_any()); - } - let decimal: Bound<'py, PyAny> = imported(py, DECIMAL_MODULE, "Decimal")?.call1((value,))?; - py.get_type::().call1((decimal,)) -} diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs index a7cdb7c2f3..7db875203d 100644 --- a/crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs @@ -1,31 +1,25 @@ //! Parsing, rendering, comparing and stepping typed cursor values. -use std::cmp::Ordering; +use crate::cursor::main::conversion::algebra_error::algebra_error; +use crate::cursor::main::scalars::as_scalar::as_scalar; use pyo3::IntoPyObject; -use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyString; use pyo3::types::PyStringMethods; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cursor::algebra::errors::CursorError; -use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::main::cursor_sort_key::cursor_sort_key as sort_key; use sqlbuild_cursor::algebra::main::exclusive_end_from_observed_max::exclusive_end_from_observed_max; -use sqlbuild_cursor::algebra::main::exclusive_to_inclusive::exclusive_to_inclusive; use sqlbuild_cursor::algebra::main::extreme_position::extreme_position; use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; -use sqlbuild_cursor::algebra::main::inclusive_to_exclusive::inclusive_to_exclusive; use sqlbuild_cursor::algebra::main::next_boundary::next_boundary; use sqlbuild_cursor::algebra::main::normalize_timestamp_text::normalize_timestamp_text; use sqlbuild_cursor::algebra::main::parse_integer_text::parse_integer_text; -use sqlbuild_cursor::algebra::main::render::render; use sqlbuild_cursor::algebra::models::{Extreme, Grain, Scalar, SortKey}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::cursor::_helpers::conversion::algebra_error; use crate::cursor::constants::DATETIME_MODULE; use crate::cursor::models::CursorSession; use sqlbuild_python_base::boundary::main::imported::imported; @@ -35,8 +29,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; const DECIMAL_MODULE: &str = "decimal"; -const INTEGER_CURSOR: &str = "integer"; -const TIMESTAMP_CURSOR: &str = "timestamp"; +pub(crate) const INTEGER_CURSOR: &str = "integer"; +pub(crate) const TIMESTAMP_CURSOR: &str = "timestamp"; /// `CursorAlgebraError(message)` raised from `cause`. fn chained(py: Python<'_>, message: String, cause: PyErr) -> PyErr { @@ -46,7 +40,7 @@ fn chained(py: Python<'_>, message: String, cause: PyErr) -> PyErr { } /// `IntegerValue` parsed from `str(raw)` the way `int(Decimal(str(raw)))` reads it. -fn parse_integer<'py>(raw: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn parse_integer<'py>(raw: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = raw.py(); let text: String = raw.str()?.to_str()?.to_owned(); if let Some(value) = parse_integer_text(&text) { @@ -93,7 +87,7 @@ fn integer_failure(py: Python<'_>, error: &PyErr) -> PyResult { } /// `value.astimezone(UTC)` for an aware `datetime`, else `value`. -fn utc_normalized<'py>(value: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn utc_normalized<'py>(value: &Bound<'py, PyAny>) -> PyResult> { if native_field(value, "tzinfo")?.is_none() { return Ok(value.clone()); } @@ -104,7 +98,10 @@ fn utc_normalized<'py>(value: &Bound<'py, PyAny>) -> PyResult> } /// A `DateValue` or `TimestampValue` parsed from warehouse timestamp text. -fn parse_timestamp_text<'py>(raw: &Bound<'py, PyAny>, text: &str) -> PyResult> { +pub(crate) fn parse_timestamp_text<'py>( + raw: &Bound<'py, PyAny>, + text: &str, +) -> PyResult> { let py: Python<'py> = raw.py(); let normalized: String = normalize_timestamp_text(text); let as_date = imported(py, DATETIME_MODULE, "date")? @@ -137,102 +134,12 @@ fn parse_timestamp_text<'py>(raw: &Bound<'py, PyAny>, text: &str) -> PyResult( - raw: &Bound<'py, PyAny>, - cursor_type: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = raw.py(); - if cursor_type.eq(INTEGER_CURSOR)? { - return parse_integer(raw); - } - if !cursor_type.eq(TIMESTAMP_CURSOR)? { - return Err(algebra_error( - py, - &format!("unsupported cursor type: {}", cursor_type.str()?), - )); - } - if raw.is_instance(&imported(py, DATETIME_MODULE, "datetime")?)? { - let value: Bound<'py, PyAny> = utc_normalized(raw)?; - return new_record_with(py, RecordId::TimestampValue, &[("value", value)]); - } - if raw.is_instance(&imported(py, DATETIME_MODULE, "date")?)? { - return new_record_with(py, RecordId::DateValue, &[("value", raw.clone())]); - } - let Ok(text) = raw.downcast::() else { - return Err(algebra_error( - py, - &format!("invalid timestamp cursor value: {}", raw.str()?), - )); - }; - parse_timestamp_text(raw, text.to_str()?) -} - -/// `raw` unchanged when it is already a typed value, else parsed. -pub(crate) fn as_scalar<'py>( - session: &CursorSession<'py>, - raw: &Bound<'py, PyAny>, - cursor_type: &Bound<'py, PyAny>, -) -> PyResult { - if session.scalar_typed(raw)? { - return session.scalar(raw); - } - session.scalar(&parse_value(raw, cursor_type)?) -} - -/// One cursor scalar, preserving date-ness and timestamp offsets. -#[pyfunction(name = "cursor_parse")] -#[pyo3(signature = (*, raw, cursor_type))] -pub(crate) fn parse<'py>( - raw: &Bound<'py, PyAny>, - cursor_type: &Bound<'py, PyAny>, -) -> PyResult> { - parse_value(raw, cursor_type) -} - -/// Discovery values that may be dates or timestamps, or `None` when they do not parse. -#[pyfunction(name = "cursor_try_parse")] -#[pyo3(signature = (*, raw, cursor_type))] -pub(crate) fn try_parse<'py>( - raw: &Bound<'py, PyAny>, - cursor_type: &Bound<'py, PyAny>, -) -> PyResult>> { - let py: Python<'py> = raw.py(); - match parse_value(raw, cursor_type) { - Ok(value) => Ok(Some(value)), - Err(error) - if error.is_instance_of::(py) - || error.is_instance_of::(py) => - { - Ok(None) - } - Err(error) => Err(error), - } -} - -/// A scalar in the existing cursor output format. -#[pyfunction(name = "cursor_render")] -#[pyo3(signature = (*, value))] -pub(crate) fn render_value(value: &Bound<'_, PyAny>) -> PyResult { - let session: CursorSession<'_> = CursorSession::new(value.py())?; - let scalar: Scalar = session.scalar(value)?; - session.checked(render(&scalar, &session)) -} - -/// A sentinel's token, or a scalar rendered. -#[pyfunction(name = "cursor_sentinel_to_token")] -#[pyo3(signature = (*, sentinel))] -pub(crate) fn sentinel_to_token(sentinel: &Bound<'_, PyAny>) -> PyResult { - if sentinel.is_instance(&host_entry(sentinel.py(), ExecutorHost::BoundSentinel)?)? { - return native_field(sentinel, "value")?.extract(); - } - render_value(sentinel) -} - /// The sentinel a SQL token names, if it names one. #[pyfunction(name = "cursor_sentinel_from_token")] #[pyo3(signature = (*, token))] -fn sentinel_from_token<'py>(token: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn sentinel_from_token<'py>( + token: &Bound<'py, PyAny>, +) -> PyResult>> { let py: Python<'py> = token.py(); match host_entry(py, ExecutorHost::BoundSentinel)?.call1((token,)) { Ok(sentinel) => Ok(Some(sentinel)), @@ -241,19 +148,6 @@ fn sentinel_from_token<'py>(token: &Bound<'py, PyAny>) -> PyResult, right: &Bound<'_, PyAny>) -> PyResult { - let session: CursorSession<'_> = CursorSession::new(left.py())?; - let (left, right): (Scalar, Scalar) = (session.scalar(left)?, session.scalar(right)?); - Ok(match session.checked(compare(&left, &right, &session))? { - Ordering::Less => -1, - Ordering::Equal => 0, - Ordering::Greater => 1, - }) -} - fn extreme_bound<'py>( values: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, @@ -280,7 +174,7 @@ fn extreme_bound<'py>( /// The numerically or temporally largest raw bound. #[pyfunction(name = "cursor_max_bound")] #[pyo3(signature = (*, values, cursor_type))] -fn max_bound<'py>( +pub(crate) fn max_bound<'py>( values: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult> { @@ -290,7 +184,7 @@ fn max_bound<'py>( /// The numerically or temporally smallest raw bound. #[pyfunction(name = "cursor_min_bound")] #[pyo3(signature = (*, values, cursor_type))] -fn min_bound<'py>( +pub(crate) fn min_bound<'py>( values: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult> { @@ -298,7 +192,7 @@ fn min_bound<'py>( } /// A scalar stepped by one grain-dependent operation. -fn stepped<'py>( +pub(crate) fn stepped<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, step: impl FnOnce(&Scalar, Option) -> Result, @@ -312,7 +206,7 @@ fn stepped<'py>( /// The aligned boundary at or before a cursor value. #[pyfunction(name = "cursor_floor_to_grain")] #[pyo3(signature = (*, value, grain))] -fn floor_value<'py>( +pub(crate) fn floor_value<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { @@ -324,37 +218,17 @@ fn floor_value<'py>( /// The boundary one grain after an aligned cursor value. #[pyfunction(name = "cursor_next_boundary")] #[pyo3(signature = (*, value, grain))] -fn next_value<'py>( +pub(crate) fn next_value<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { stepped(value, grain, next_boundary) } -/// An operator-declared inclusive bound advanced by one cursor unit. -#[pyfunction(name = "cursor_inclusive_to_exclusive")] -#[pyo3(signature = (*, value, grain))] -pub(crate) fn to_exclusive<'py>( - value: &Bound<'py, PyAny>, - grain: &Bound<'py, PyAny>, -) -> PyResult> { - stepped(value, grain, inclusive_to_exclusive) -} - -/// An exclusive bound moved back by one cursor unit. -#[pyfunction(name = "cursor_exclusive_to_inclusive")] -#[pyo3(signature = (*, value, grain))] -pub(crate) fn to_inclusive<'py>( - value: &Bound<'py, PyAny>, - grain: &Bound<'py, PyAny>, -) -> PyResult> { - stepped(value, grain, exclusive_to_inclusive) -} - /// The exclusive end of the partition holding an observed maximum. #[pyfunction(name = "cursor_exclusive_end_from_observed_max")] #[pyo3(signature = (*, value, grain))] -fn observed_end<'py>( +pub(crate) fn observed_end<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { @@ -367,20 +241,3 @@ fn observed_end<'py>( ))?; session.python_scalar(&end) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse, module)?)?; - module.add_function(wrap_pyfunction!(try_parse, module)?)?; - module.add_function(wrap_pyfunction!(render_value, module)?)?; - module.add_function(wrap_pyfunction!(sentinel_to_token, module)?)?; - module.add_function(wrap_pyfunction!(sentinel_from_token, module)?)?; - module.add_function(wrap_pyfunction!(compare_values, module)?)?; - module.add_function(wrap_pyfunction!(max_bound, module)?)?; - module.add_function(wrap_pyfunction!(min_bound, module)?)?; - module.add_function(wrap_pyfunction!(floor_value, module)?)?; - module.add_function(wrap_pyfunction!(next_value, module)?)?; - module.add_function(wrap_pyfunction!(to_exclusive, module)?)?; - module.add_function(wrap_pyfunction!(to_inclusive, module)?)?; - module.add_function(wrap_pyfunction!(observed_end, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/cursor/_helpers/values.rs b/crates/sqlbuild-python-front/src/cursor/_helpers/values.rs index de30f9f798..52b8214bb3 100644 --- a/crates/sqlbuild-python-front/src/cursor/_helpers/values.rs +++ b/crates/sqlbuild-python-front/src/cursor/_helpers/values.rs @@ -1,11 +1,10 @@ //! Python `date`, `datetime` and `timedelta` objects read into and built from cursor values. use crate::cursor::constants::DATETIME_MODULE; -use pyo3::IntoPyObject; +use crate::cursor::main::values::replaced::replaced; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_cursor::algebra::models::{CivilDate, LocalDateTime}; -use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -21,17 +20,6 @@ pub(crate) fn civil_date(value: &Bound<'_, PyAny>) -> PyResult { }) } -/// The wall-clock fields of a `datetime`. -pub(crate) fn local_date_time(value: &Bound<'_, PyAny>) -> PyResult { - Ok(LocalDateTime { - date: civil_date(value)?, - hour: native_field(value, "hour")?.extract()?, - minute: native_field(value, "minute")?.extract()?, - second: native_field(value, "second")?.extract()?, - microsecond: native_field(value, "microsecond")?.extract()?, - }) -} - /// A `timedelta` as whole microseconds. pub(crate) fn timedelta_micros(delta: &Bound<'_, PyAny>) -> PyResult { let days: i64 = native_field(delta, "days")?.extract()?; @@ -40,31 +28,6 @@ pub(crate) fn timedelta_micros(delta: &Bound<'_, PyAny>) -> PyResult { Ok((days * SECONDS_PER_DAY + seconds) * MICROS_PER_SECOND + microseconds) } -/// `template.replace(...)` with the wall time and fold of `local`, keeping type and `tzinfo`. -pub(crate) fn replaced<'py>( - template: &Bound<'py, PyAny>, - local: &LocalDateTime, - fold: bool, -) -> PyResult> { - let py: Python<'py> = template.py(); - call_with( - &native_field(template, "replace")?, - &[ - ("year", local.date.year.into_pyobject(py)?.into_any()), - ("month", local.date.month.into_pyobject(py)?.into_any()), - ("day", local.date.day.into_pyobject(py)?.into_any()), - ("hour", local.hour.into_pyobject(py)?.into_any()), - ("minute", local.minute.into_pyobject(py)?.into_any()), - ("second", local.second.into_pyobject(py)?.into_any()), - ( - "microsecond", - local.microsecond.into_pyobject(py)?.into_any(), - ), - ("fold", u8::from(fold).into_pyobject(py)?.into_any()), - ], - ) -} - /// A naive `datetime` holding `local`. pub(crate) fn naive_date_time<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python-front/src/cursor/main/conversion/algebra_error.rs b/crates/sqlbuild-python-front/src/cursor/main/conversion/algebra_error.rs index 67da83f9c4..58efc324ca 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/conversion/algebra_error.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/conversion/algebra_error.rs @@ -1,11 +1,10 @@ //! `CursorAlgebraError(message)`. -use crate::cursor::_helpers::conversion; -use pyo3::prelude::PyErr; -use pyo3::prelude::Python; +use pyo3::prelude::{PyErr, Python}; +use sqlbuild_host::entries::models::FacadeHost; +use sqlbuild_python_base::native_host::main::host_error::host_error; /// `CursorAlgebraError(message)`. -#[inline] pub fn algebra_error(py: Python<'_>, message: &str) -> PyErr { - conversion::algebra_error(py, message) + host_error(py, FacadeHost::CursorAlgebraError, message) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/durations/parse_cursor_duration.rs b/crates/sqlbuild-python-front/src/cursor/main/durations/parse_cursor_duration.rs index 932140f66b..eac5496ad9 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/durations/parse_cursor_duration.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/durations/parse_cursor_duration.rs @@ -1,16 +1,32 @@ //! `(years, months, days, hours, minutes, seconds)` of a duration like `1y6mo` or `30d`. -use crate::cursor::_helpers::durations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyTuple; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyInt, PyTuple}; +use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; +use sqlbuild_cursor::algebra::models::DurationDigits; /// `(years, months, days, hours, minutes, seconds)` of a duration like `1y6mo` or `30d`. -#[inline] +#[pyfunction] +#[pyo3(signature = (value))] pub fn parse_cursor_duration<'py>( py: Python<'py>, value: &str, ) -> PyResult>> { - durations::parse_cursor_duration(py, value) + let Some(digits): Option = parse_duration(value) else { + return Ok(None); + }; + let int = py.get_type::(); + let parts: Vec> = [ + digits.years, + digits.months, + digits.days, + digits.hours, + digits.minutes, + digits.seconds, + ] + .iter() + .map(|text| int.call1((text.as_str(),))) + .collect::>()?; + PyTuple::new(py, parts).map(Some) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/durations/register_durations.rs b/crates/sqlbuild-python-front/src/cursor/main/durations/register_durations.rs index 77d1ebdb29..516c11fa7b 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/durations/register_durations.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/durations/register_durations.rs @@ -1,12 +1,13 @@ //! Register the durations functions and classes. -use crate::cursor::_helpers::durations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::cursor::main::durations::parse_cursor_duration::parse_cursor_duration; +use crate::cursor::main::durations::shift_by_cursor_duration::shift_by_cursor_duration; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the durations functions and classes. -#[inline] pub fn register_durations(module: &Bound<'_, PyModule>) -> PyResult<()> { - durations::register(module) + module.add_function(wrap_pyfunction!(parse_cursor_duration, module)?)?; + module.add_function(wrap_pyfunction!(shift_by_cursor_duration, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/durations/shift_by_cursor_duration.rs b/crates/sqlbuild-python-front/src/cursor/main/durations/shift_by_cursor_duration.rs index 9037da5d56..9065b91a10 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/durations/shift_by_cursor_duration.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/durations/shift_by_cursor_duration.rs @@ -1,16 +1,17 @@ //! `moment` moved forward (`direction` 1) or back (`direction` -1) by `duration`. -use crate::cursor::_helpers::durations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::durations::duration_span; +use crate::cursor::main::durations::shifted_moment::shifted_moment; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// `moment` moved forward (`direction` 1) or back (`direction` -1) by `duration`. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, moment, duration, direction))] pub fn shift_by_cursor_duration<'py>( moment: &Bound<'py, PyAny>, duration: &Bound<'py, PyAny>, direction: i64, ) -> PyResult> { - durations::shift_by_cursor_duration(moment, duration, direction) + shifted_moment(moment, &duration_span(duration)?, direction) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/durations/shifted_moment.rs b/crates/sqlbuild-python-front/src/cursor/main/durations/shifted_moment.rs index c92de35127..68f8cfec0d 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/durations/shifted_moment.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/durations/shifted_moment.rs @@ -1,17 +1,45 @@ //! `moment` (a `date` or `datetime`) moved by `span` in `direction`, keeping its type and `tzinfo`. -use crate::cursor::_helpers::durations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use sqlbuild_cursor::algebra::models::DurationSpan; +use crate::cursor::_helpers::values::civil_date; +use crate::cursor::constants::DATETIME_MODULE; +use crate::cursor::main::values::local_date_time::local_date_time; +use crate::cursor::main::values::replaced::replaced; +use crate::cursor::models::CursorSession; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_cursor::algebra::main::shift_by_duration::shift_by_duration; +use sqlbuild_cursor::algebra::models::{DurationSpan, Moment}; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `moment` (a `date` or `datetime`) moved by `span` in `direction`, keeping its type and `tzinfo`. -#[inline] pub fn shifted_moment<'py>( moment: &Bound<'py, PyAny>, span: &DurationSpan, direction: i64, ) -> PyResult> { - durations::shifted_moment(moment, span, direction) + let py: Python<'py> = moment.py(); + let session: CursorSession<'py> = CursorSession::new(py)?; + let is_timestamp: bool = moment.is_instance(&imported(py, DATETIME_MODULE, "datetime")?)?; + let typed: Moment = if is_timestamp { + Moment::Timestamp { + local: local_date_time(moment)?, + fold: native_field(moment, "fold")?.extract::()? != 0, + } + } else { + Moment::Date(civil_date(moment)?) + }; + let shifted: Moment = session.checked(shift_by_duration(typed, span, direction))?; + match shifted { + Moment::Timestamp { local, fold } => replaced(moment, &local, fold), + Moment::Date(date) => call_with( + &native_field(moment, "replace")?, + &[ + ("year", date.year.into_pyobject(py)?.into_any()), + ("month", date.month.into_pyobject(py)?.into_any()), + ("day", date.day.into_pyobject(py)?.into_any()), + ], + ), + } } diff --git a/crates/sqlbuild-python-front/src/cursor/main/intervals/interval_validation.rs b/crates/sqlbuild-python-front/src/cursor/main/intervals/interval_validation.rs index 549343ce56..f7fbc613cd 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/intervals/interval_validation.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/intervals/interval_validation.rs @@ -1,16 +1,24 @@ //! Raise `CursorAlgebraError` unless the ends form a valid aligned interval. -use crate::cursor::_helpers::intervals; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::models::CursorSession; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_cursor::algebra::main::validate_interval::validate_interval; +use sqlbuild_cursor::algebra::models::Interval; /// Raise `CursorAlgebraError` unless the ends form a valid aligned interval. -#[inline] +#[pyfunction(name = "validate_cursor_interval")] +#[pyo3(signature = (*, start, end, grain))] pub fn interval_validation( start: &Bound<'_, PyAny>, end: &Bound<'_, PyAny>, grain: &Bound<'_, PyAny>, ) -> PyResult<()> { - intervals::interval_validation(start, end, grain) + let session: CursorSession<'_> = CursorSession::new(start.py())?; + let interval = Interval { + start: session.scalar(start)?, + end: session.scalar(end)?, + grain: session.grain(grain)?, + }; + session.checked(validate_interval(&interval, &session)) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/intervals/partition.rs b/crates/sqlbuild-python-front/src/cursor/main/intervals/partition.rs index 7509cc126b..1c7393d999 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/intervals/partition.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/intervals/partition.rs @@ -1,15 +1,20 @@ //! The canonical partition containing an observed physical value. -use crate::cursor::_helpers::intervals; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::models::CursorSession; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_cursor::algebra::main::observed_partition::observed_partition; +use sqlbuild_cursor::algebra::models::{Interval, Scalar}; /// The canonical partition containing an observed physical value. -#[inline] +#[pyfunction(name = "cursor_observed_partition")] +#[pyo3(signature = (*, value, grain))] pub fn partition<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { - intervals::partition(value, grain) + let session: CursorSession<'py> = CursorSession::new(value.py())?; + let scalar: Scalar = session.scalar(value)?; + let interval: Interval = session.checked(observed_partition(&scalar, session.grain(grain)?))?; + session.python_interval(&interval) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/intervals/register_intervals.rs b/crates/sqlbuild-python-front/src/cursor/main/intervals/register_intervals.rs index 33e94ed6ad..9e4a224eb8 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/intervals/register_intervals.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/intervals/register_intervals.rs @@ -1,12 +1,25 @@ //! Register the intervals functions and classes. -use crate::cursor::_helpers::intervals; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::intervals::cap_end; +use crate::cursor::_helpers::intervals::cap_start; +use crate::cursor::_helpers::intervals::clamp_intervals; +use crate::cursor::_helpers::intervals::merge_intervals; +use crate::cursor::_helpers::intervals::merge_raw_intervals; +use crate::cursor::_helpers::intervals::split; +use crate::cursor::main::intervals::interval_validation::interval_validation; +use crate::cursor::main::intervals::partition::partition; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the intervals functions and classes. -#[inline] pub fn register_intervals(module: &Bound<'_, PyModule>) -> PyResult<()> { - intervals::register(module) + module.add_function(wrap_pyfunction!(interval_validation, module)?)?; + module.add_function(wrap_pyfunction!(partition, module)?)?; + module.add_function(wrap_pyfunction!(clamp_intervals, module)?)?; + module.add_function(wrap_pyfunction!(merge_intervals, module)?)?; + module.add_function(wrap_pyfunction!(merge_raw_intervals, module)?)?; + module.add_function(wrap_pyfunction!(split, module)?)?; + module.add_function(wrap_pyfunction!(cap_start, module)?)?; + module.add_function(wrap_pyfunction!(cap_end, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_integer.rs b/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_integer.rs index a7fc01d88c..bc24492456 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_integer.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_integer.rs @@ -1,12 +1,34 @@ //! An integer override as an `int`; text is read the way `int(Decimal(text))` reads it. -use crate::cursor::_helpers::overrides; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::overrides::DECIMAL_MODULE; +use crate::cursor::_helpers::overrides::TIMESTAMP_AS_INTEGER; +use crate::cursor::_helpers::overrides::instance_of; +use crate::cursor::main::conversion::algebra_error::algebra_error; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyInt, PyString, PyStringMethods}; +use sqlbuild_cursor::algebra::main::parse_integer_text::parse_integer_text; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// An integer override as an `int`; text is read the way `int(Decimal(text))` reads it. -#[inline] pub fn cursor_integer<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - overrides::cursor_integer(value) + let py: Python<'py> = value.py(); + if value.is_none() { + return Ok(value.clone()); + } + if instance_of(value, RecordId::IntegerValue)? { + return native_field(value, "value"); + } + if instance_of(value, RecordId::DateValue)? || instance_of(value, RecordId::TimestampValue)? { + return Err(algebra_error(py, TIMESTAMP_AS_INTEGER)); + } + if let Ok(text) = value.downcast::() + && let Some(parsed) = parse_integer_text(text.to_str()?) + { + return Ok(parsed.into_pyobject(py)?.into_any()); + } + let decimal: Bound<'py, PyAny> = imported(py, DECIMAL_MODULE, "Decimal")?.call1((value,))?; + py.get_type::().call1((decimal,)) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_timestamp.rs b/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_timestamp.rs index 193365f411..ab1a9e892a 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_timestamp.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_timestamp.rs @@ -1,12 +1,30 @@ //! A timestamp override as a `datetime`; dates start at midnight and text is ISO 8601. -use crate::cursor::_helpers::overrides; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::overrides::INTEGER_AS_TIMESTAMP; +use crate::cursor::_helpers::overrides::instance_of; +use crate::cursor::constants::DATETIME_MODULE; +use crate::cursor::main::conversion::algebra_error::algebra_error; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// A timestamp override as a `datetime`; dates start at midnight and text is ISO 8601. -#[inline] pub fn cursor_timestamp<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - overrides::cursor_timestamp(value) + let py: Python<'py> = value.py(); + if value.is_none() { + return Ok(value.clone()); + } + if instance_of(value, RecordId::TimestampValue)? { + return native_field(value, "value"); + } + let datetime: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; + if instance_of(value, RecordId::DateValue)? { + let midnight: Bound<'py, PyAny> = native_field(&datetime, "min")?.call_method0("time")?; + return datetime.call_method1("combine", (native_field(value, "value")?, midnight)); + } + if instance_of(value, RecordId::IntegerValue)? { + return Err(algebra_error(py, INTEGER_AS_TIMESTAMP)); + } + datetime.call_method1("fromisoformat", (value,)) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/as_scalar.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/as_scalar.rs index 56617ed205..987d827623 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/as_scalar.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/as_scalar.rs @@ -1,18 +1,18 @@ //! `raw` unchanged when it is already a typed value, else parsed. -use crate::cursor::_helpers::scalars; +use crate::cursor::main::scalars::parse_value::parse_value; use crate::cursor::models::CursorSession; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_cursor::algebra::models::Scalar; /// `raw` unchanged when it is already a typed value, else parsed. -#[inline] pub fn as_scalar<'py>( session: &CursorSession<'py>, raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult { - scalars::as_scalar(session, raw, cursor_type) + if session.scalar_typed(raw)? { + return session.scalar(raw); + } + session.scalar(&parse_value(raw, cursor_type)?) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/compare_values.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/compare_values.rs index d8a1465d67..94c5f12964 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/compare_values.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/compare_values.rs @@ -1,12 +1,21 @@ //! `-1`, `0` or `1` as `left` orders against `right`. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::models::CursorSession; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_cursor::algebra::main::compare::compare; +use sqlbuild_cursor::algebra::models::Scalar; +use std::cmp::Ordering; /// `-1`, `0` or `1` as `left` orders against `right`. -#[inline] +#[pyfunction(name = "cursor_compare")] +#[pyo3(signature = (*, left, right))] pub fn compare_values(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult { - scalars::compare_values(left, right) + let session: CursorSession<'_> = CursorSession::new(left.py())?; + let (left, right): (Scalar, Scalar) = (session.scalar(left)?, session.scalar(right)?); + Ok(match session.checked(compare(&left, &right, &session))? { + Ordering::Less => -1, + Ordering::Equal => 0, + Ordering::Greater => 1, + }) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/parse.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/parse.rs index b9d0b5af14..9c83a032c5 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/parse.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/parse.rs @@ -1,15 +1,15 @@ //! One cursor scalar, preserving date-ness and timestamp offsets. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::main::scalars::parse_value::parse_value; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// One cursor scalar, preserving date-ness and timestamp offsets. -#[inline] +#[pyfunction(name = "cursor_parse")] +#[pyo3(signature = (*, raw, cursor_type))] pub fn parse<'py>( raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult> { - scalars::parse(raw, cursor_type) + parse_value(raw, cursor_type) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/parse_value.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/parse_value.rs index 198f84f246..b0f2dbc60b 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/parse_value.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/parse_value.rs @@ -1,15 +1,46 @@ //! One cursor value parsed from its boundary representation, keeping date-ness and offsets. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::scalars::INTEGER_CURSOR; +use crate::cursor::_helpers::scalars::TIMESTAMP_CURSOR; +use crate::cursor::_helpers::scalars::parse_integer; +use crate::cursor::_helpers::scalars::parse_timestamp_text; +use crate::cursor::_helpers::scalars::utc_normalized; +use crate::cursor::constants::DATETIME_MODULE; +use crate::cursor::main::conversion::algebra_error::algebra_error; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use pyo3::types::PyStringMethods; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; /// One cursor value parsed from its boundary representation, keeping date-ness and offsets. -#[inline] pub fn parse_value<'py>( raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult> { - scalars::parse_value(raw, cursor_type) + let py: Python<'py> = raw.py(); + if cursor_type.eq(INTEGER_CURSOR)? { + return parse_integer(raw); + } + if !cursor_type.eq(TIMESTAMP_CURSOR)? { + return Err(algebra_error( + py, + &format!("unsupported cursor type: {}", cursor_type.str()?), + )); + } + if raw.is_instance(&imported(py, DATETIME_MODULE, "datetime")?)? { + let value: Bound<'py, PyAny> = utc_normalized(raw)?; + return new_record_with(py, RecordId::TimestampValue, &[("value", value)]); + } + if raw.is_instance(&imported(py, DATETIME_MODULE, "date")?)? { + return new_record_with(py, RecordId::DateValue, &[("value", raw.clone())]); + } + let Ok(text) = raw.downcast::() else { + return Err(algebra_error( + py, + &format!("invalid timestamp cursor value: {}", raw.str()?), + )); + }; + parse_timestamp_text(raw, text.to_str()?) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/register_scalars.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/register_scalars.rs index 5d54f46a04..e4b6eca099 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/register_scalars.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/register_scalars.rs @@ -1,12 +1,35 @@ //! Register the scalars functions and classes. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::scalars::floor_value; +use crate::cursor::_helpers::scalars::max_bound; +use crate::cursor::_helpers::scalars::min_bound; +use crate::cursor::_helpers::scalars::next_value; +use crate::cursor::_helpers::scalars::observed_end; +use crate::cursor::_helpers::scalars::sentinel_from_token; +use crate::cursor::main::scalars::compare_values::compare_values; +use crate::cursor::main::scalars::parse::parse; +use crate::cursor::main::scalars::render_value::render_value; +use crate::cursor::main::scalars::sentinel_to_token::sentinel_to_token; +use crate::cursor::main::scalars::to_exclusive::to_exclusive; +use crate::cursor::main::scalars::to_inclusive::to_inclusive; +use crate::cursor::main::scalars::try_parse::try_parse; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scalars functions and classes. -#[inline] pub fn register_scalars(module: &Bound<'_, PyModule>) -> PyResult<()> { - scalars::register(module) + module.add_function(wrap_pyfunction!(parse, module)?)?; + module.add_function(wrap_pyfunction!(try_parse, module)?)?; + module.add_function(wrap_pyfunction!(render_value, module)?)?; + module.add_function(wrap_pyfunction!(sentinel_to_token, module)?)?; + module.add_function(wrap_pyfunction!(sentinel_from_token, module)?)?; + module.add_function(wrap_pyfunction!(compare_values, module)?)?; + module.add_function(wrap_pyfunction!(max_bound, module)?)?; + module.add_function(wrap_pyfunction!(min_bound, module)?)?; + module.add_function(wrap_pyfunction!(floor_value, module)?)?; + module.add_function(wrap_pyfunction!(next_value, module)?)?; + module.add_function(wrap_pyfunction!(to_exclusive, module)?)?; + module.add_function(wrap_pyfunction!(to_inclusive, module)?)?; + module.add_function(wrap_pyfunction!(observed_end, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/render_value.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/render_value.rs index cc7c06fb71..69063d3b68 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/render_value.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/render_value.rs @@ -1,12 +1,16 @@ //! A scalar in the existing cursor output format. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::models::CursorSession; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_cursor::algebra::main::render::render; +use sqlbuild_cursor::algebra::models::Scalar; /// A scalar in the existing cursor output format. -#[inline] +#[pyfunction(name = "cursor_render")] +#[pyo3(signature = (*, value))] pub fn render_value(value: &Bound<'_, PyAny>) -> PyResult { - scalars::render_value(value) + let session: CursorSession<'_> = CursorSession::new(value.py())?; + let scalar: Scalar = session.scalar(value)?; + session.checked(render(&scalar, &session)) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/sentinel_to_token.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/sentinel_to_token.rs index fd2abf4730..a6d10c2234 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/sentinel_to_token.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/sentinel_to_token.rs @@ -1,12 +1,18 @@ //! A sentinel's token, or a scalar rendered. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::main::scalars::render_value::render_value; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// A sentinel's token, or a scalar rendered. -#[inline] +#[pyfunction(name = "cursor_sentinel_to_token")] +#[pyo3(signature = (*, sentinel))] pub fn sentinel_to_token(sentinel: &Bound<'_, PyAny>) -> PyResult { - scalars::sentinel_to_token(sentinel) + if sentinel.is_instance(&host_entry(sentinel.py(), ExecutorHost::BoundSentinel)?)? { + return native_field(sentinel, "value")?.extract(); + } + render_value(sentinel) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/to_exclusive.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/to_exclusive.rs index 3f71c47689..45793135f2 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/to_exclusive.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/to_exclusive.rs @@ -1,15 +1,16 @@ //! An operator-declared inclusive bound advanced by one cursor unit. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::scalars::stepped; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_cursor::algebra::main::inclusive_to_exclusive::inclusive_to_exclusive; /// An operator-declared inclusive bound advanced by one cursor unit. -#[inline] +#[pyfunction(name = "cursor_inclusive_to_exclusive")] +#[pyo3(signature = (*, value, grain))] pub fn to_exclusive<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { - scalars::to_exclusive(value, grain) + stepped(value, grain, inclusive_to_exclusive) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/to_inclusive.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/to_inclusive.rs index 49da0d7d19..766d74929b 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/to_inclusive.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/to_inclusive.rs @@ -1,15 +1,16 @@ //! An exclusive bound moved back by one cursor unit. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::scalars::stepped; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_cursor::algebra::main::exclusive_to_inclusive::exclusive_to_inclusive; /// An exclusive bound moved back by one cursor unit. -#[inline] +#[pyfunction(name = "cursor_exclusive_to_inclusive")] +#[pyo3(signature = (*, value, grain))] pub fn to_inclusive<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { - scalars::to_inclusive(value, grain) + stepped(value, grain, exclusive_to_inclusive) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/scalars/try_parse.rs b/crates/sqlbuild-python-front/src/cursor/main/scalars/try_parse.rs index aac8b2cfb1..dcf88432f4 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/scalars/try_parse.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/scalars/try_parse.rs @@ -1,15 +1,26 @@ //! Discovery values that may be dates or timestamps, or `None` when they do not parse. -use crate::cursor::_helpers::scalars; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::main::scalars::parse_value::parse_value; +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; /// Discovery values that may be dates or timestamps, or `None` when they do not parse. -#[inline] +#[pyfunction(name = "cursor_try_parse")] +#[pyo3(signature = (*, raw, cursor_type))] pub fn try_parse<'py>( raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult>> { - scalars::try_parse(raw, cursor_type) + let py: Python<'py> = raw.py(); + match parse_value(raw, cursor_type) { + Ok(value) => Ok(Some(value)), + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + Ok(None) + } + Err(error) => Err(error), + } } diff --git a/crates/sqlbuild-python-front/src/cursor/main/values/local_date_time.rs b/crates/sqlbuild-python-front/src/cursor/main/values/local_date_time.rs index 4a4809c85d..c0dde89480 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/values/local_date_time.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/values/local_date_time.rs @@ -1,13 +1,17 @@ //! The wall-clock fields of a `datetime`. -use crate::cursor::_helpers::values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::_helpers::values::civil_date; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_cursor::algebra::models::LocalDateTime; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The wall-clock fields of a `datetime`. -#[inline] pub fn local_date_time(value: &Bound<'_, PyAny>) -> PyResult { - values::local_date_time(value) + Ok(LocalDateTime { + date: civil_date(value)?, + hour: native_field(value, "hour")?.extract()?, + minute: native_field(value, "minute")?.extract()?, + second: native_field(value, "second")?.extract()?, + microsecond: native_field(value, "microsecond")?.extract()?, + }) } diff --git a/crates/sqlbuild-python-front/src/cursor/main/values/replaced.rs b/crates/sqlbuild-python-front/src/cursor/main/values/replaced.rs index c6d6e8c946..b910208924 100644 --- a/crates/sqlbuild-python-front/src/cursor/main/values/replaced.rs +++ b/crates/sqlbuild-python-front/src/cursor/main/values/replaced.rs @@ -1,17 +1,32 @@ //! `template.replace(...)` with the wall time and fold of `local`, keeping type and `tzinfo`. -use crate::cursor::_helpers::values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use sqlbuild_cursor::algebra::models::LocalDateTime; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `template.replace(...)` with the wall time and fold of `local`, keeping type and `tzinfo`. -#[inline] pub fn replaced<'py>( template: &Bound<'py, PyAny>, local: &LocalDateTime, fold: bool, ) -> PyResult> { - values::replaced(template, local, fold) + let py: Python<'py> = template.py(); + call_with( + &native_field(template, "replace")?, + &[ + ("year", local.date.year.into_pyobject(py)?.into_any()), + ("month", local.date.month.into_pyobject(py)?.into_any()), + ("day", local.date.day.into_pyobject(py)?.into_any()), + ("hour", local.hour.into_pyobject(py)?.into_any()), + ("minute", local.minute.into_pyobject(py)?.into_any()), + ("second", local.second.into_pyobject(py)?.into_any()), + ( + "microsecond", + local.microsecond.into_pyobject(py)?.into_any(), + ), + ("fold", u8::from(fold).into_pyobject(py)?.into_any()), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/cursor/models.rs b/crates/sqlbuild-python-front/src/cursor/models.rs index 7a7a17b513..d93e6b7856 100644 --- a/crates/sqlbuild-python-front/src/cursor/models.rs +++ b/crates/sqlbuild-python-front/src/cursor/models.rs @@ -1,14 +1,14 @@ //! Public records of cursor. -use crate::cursor::_helpers::conversion::algebra_error; use crate::cursor::_helpers::values::civil_date; -use crate::cursor::_helpers::values::local_date_time; use crate::cursor::_helpers::values::naive_date_time; use crate::cursor::_helpers::values::plain_date; -use crate::cursor::_helpers::values::replaced; use crate::cursor::_helpers::values::timedelta_micros; use crate::cursor::constants::DATETIME_MODULE; use crate::cursor::constants::GRAIN_NAMES; +use crate::cursor::main::conversion::algebra_error::algebra_error; +use crate::cursor::main::values::local_date_time::local_date_time; +use crate::cursor::main::values::replaced::replaced; use pyo3::IntoPyObject; use pyo3::exceptions::PyOverflowError; use pyo3::exceptions::PyTypeError; diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/failures/guidance.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/failures/guidance.rs index 7948942caa..1e14af603e 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/failures/guidance.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/failures/guidance.rs @@ -1,61 +1,26 @@ //! Setting guidance shown by diagnostics that depend on project settings. +use crate::discovery::main::failures::sql_analysis_guidance::sql_analysis_guidance; use pyo3::exceptions::PyTypeError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyBoolMethods, PyDict, PyInt, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_config::project::main::explicit_references_help::explicit_references_help as references_help; -use sqlbuild_config::project::main::microbatch_guidance::microbatch_guidance as concurrency_guidance; -use sqlbuild_config::project::main::setting_help_text::setting_help_text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyBool, PyBoolMethods, PyInt, PyString}; use sqlbuild_config::project::main::snippet_help_text::snippet_help_text; -use sqlbuild_config::project::main::sql_analysis_off_guidance::sql_analysis_off_guidance as analysis_off; -use sqlbuild_config::project::models::{SettingSite, SettingText}; +use sqlbuild_config::project::models::SettingText; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -const SQL_ANALYSIS_KEY: &str = "sql_analysis"; - -/// The note and help for turning SQL analysis on, or `None` when it is effectively on. -pub(crate) fn sql_analysis_guidance( - discovered_inputs: &Bound<'_, PyAny>, - purpose: &str, -) -> PyResult> { - let local: Bound<'_, PyAny> = native_field(discovered_inputs, "local_config")?; - let local_override: bool = - native_field(&local, "setting_overrides")?.contains(SQL_ANALYSIS_KEY)?; - let owner: Bound<'_, PyAny> = if local_override { - local - } else { - native_field(discovered_inputs, "project_config")? - }; - if native_field(&native_field(&owner, "settings")?, SQL_ANALYSIS_KEY)?.is_truthy()? { - return Ok(None); - } - Ok(Some(analysis_off(local_override, purpose))) -} +pub(crate) const SQL_ANALYSIS_KEY: &str = "sql_analysis"; #[pyfunction] #[pyo3(signature = (*, discovered_inputs, purpose))] -fn sql_analysis_off_guidance( +pub(crate) fn sql_analysis_off_guidance( discovered_inputs: &Bound<'_, PyAny>, purpose: &str, ) -> PyResult> { sql_analysis_guidance(discovered_inputs, purpose) } -#[pyfunction] -pub(crate) fn microbatch_guidance() -> (String, String) { - concurrency_guidance() -} - -#[pyfunction] -#[pyo3(signature = (*, allowed))] -pub(crate) fn explicit_references_help(allowed: &str) -> String { - references_help(allowed) -} - /// A Python `bool | int | str` setting value as help text shows it. -fn setting_text(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn setting_text(value: &Bound<'_, PyAny>) -> PyResult { if let Ok(flag) = value.cast::() { return Ok(SettingText::Bool(flag.is_true())); } @@ -70,45 +35,9 @@ fn setting_text(value: &Bound<'_, PyAny>) -> PyResult { )) } -/// Render `, set this in :` followed by the exact TOML section and key. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn setting_help<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "setting_help", - options, - (&["purpose", "file_name", "section", "key", "value"], &[]), - )?; - let text = |name: &str| -> PyResult { arguments.value(name)?.extract() }; - let (file_name, section, key): (String, String, String) = - (text("file_name")?, text("section")?, text("key")?); - Ok(setting_help_text( - &text("purpose")?, - SettingSite { - file_name: &file_name, - section: §ion, - key: &key, - }, - &setting_text(&arguments.value("value")?)?, - )) -} - /// Render `, :` followed by the exact lines to write, one per line. #[pyfunction] #[pyo3(signature = (*, purpose, target, lines))] -fn snippet_help(purpose: &str, target: &str, lines: Vec) -> String { +pub(crate) fn snippet_help(purpose: &str, target: &str, lines: Vec) -> String { snippet_help_text(purpose, target, &lines) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(setting_help, module)?)?; - module.add_function(wrap_pyfunction!(snippet_help, module)?)?; - module.add_function(wrap_pyfunction!(sql_analysis_off_guidance, module)?)?; - module.add_function(wrap_pyfunction!(microbatch_guidance, module)?)?; - module.add_function(wrap_pyfunction!(explicit_references_help, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/failures/raised.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/failures/raised.rs index 51e2e48d0c..7f03682b3f 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/failures/raised.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/failures/raised.rs @@ -1,11 +1,9 @@ //! Native discovery failures as the Python exceptions discovery raises for them. use pyo3::exceptions::{PyKeyError, PyRuntimeError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyBytes, PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::models::{DiscoveryFailure, ReadFailure, StageFailure}; use sqlbuild_discovery::tree::models::ProjectTree; @@ -135,7 +133,7 @@ pub(crate) fn relative_read_error( /// The error `read_error` builds for an operating-system read failure, for the platform oracle. #[pyfunction] #[pyo3(signature = (errno, winerror, message, *, file_path))] -fn _oracle_read_error<'py>( +pub(crate) fn _oracle_read_error<'py>( errno: Option, winerror: Option, message: String, @@ -151,7 +149,3 @@ fn _oracle_read_error<'py>( .into_bound(file_path.py()) .into_any() } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(_oracle_read_error, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_declarations.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_declarations.rs index 4fe5562215..f9c864a4cb 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_declarations.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_declarations.rs @@ -1,10 +1,8 @@ //! Model-header enums and constants, and typed constant file entries, validated natively. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_model_config::declarations::errors::DeclarationFailure; use sqlbuild_model_config::declarations::main::model_constant_declarations::model_constant_declarations; use sqlbuild_model_config::declarations::main::model_enum_declarations::model_enum_declarations; @@ -190,7 +188,3 @@ pub(crate) fn typed_constant_record<'py>( Err(failure) => Err(declaration_error(py, failure)?), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(model_header_declarations, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_values.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_values.rs index d3418c38d6..22bc1ac8d7 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_values.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_values.rs @@ -1,9 +1,9 @@ //! MODEL-style header values with a bounded parse cache, and authored column locations. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use crate::sqltext::main::header_failure::header_failure; use crate::sqltext::main::model_output_columns::model_output_columns; @@ -30,7 +30,7 @@ fn header_cache(py: Python<'_>) -> Bound<'_, PyDict> { /// Parse every header the cache misses in one native batch. #[pyfunction] -fn prepare_model_header_tokens(headers: &Bound<'_, PyAny>) -> PyResult<()> { +pub(crate) fn prepare_model_header_tokens(headers: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = headers.py(); let cache: Bound<'_, PyDict> = header_cache(py); let missing: Bound<'_, PyDict> = PyDict::new(py); @@ -57,7 +57,9 @@ fn prepare_model_header_tokens(headers: &Bound<'_, PyAny>) -> PyResult<()> { /// `parse_header_values(*, header, file_path, statement_name, header_line, error_class=None)`. #[pyfunction] #[pyo3(signature = (**request))] -fn parse_header_values<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn parse_header_values<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("parse_header_values", request)?; let py: Python<'py> = request.py(); let header: Bound<'py, PyAny> = request.get("header")?; @@ -131,7 +133,7 @@ fn source_locations<'py>( /// `model_output_column_locations(*, contents, relative_path, extract_implicit_alias_columns=True)`. #[pyfunction] #[pyo3(signature = (**request))] -fn model_output_column_locations<'py>( +pub(crate) fn model_output_column_locations<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("model_output_column_locations", request)?; @@ -143,9 +145,3 @@ fn model_output_column_locations<'py>( let rows: Vec = model_output_columns(&contents, implicit)?; source_locations(rows, &request.get("relative_path")?) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(prepare_model_header_tokens, module)?)?; - module.add_function(wrap_pyfunction!(parse_header_values, module)?)?; - module.add_function(wrap_pyfunction!(model_output_column_locations, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/hook_entries.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/hook_entries.rs index 6b775040f7..556f8e94da 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/hook_entries.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/hook_entries.rs @@ -1,12 +1,11 @@ //! Model lifecycle hooks as the plain dictionaries manifests and identities record. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::discovery::main::headers::hook_entries::hook_entries; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyList}; -use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; const RELATIVE_PATH_FIELD: &str = "relative_path"; const SQL_HOOK_FIELDS: [&str; 6] = [ @@ -18,7 +17,10 @@ const SQL_HOOK_FIELDS: [&str; 6] = [ "description", ]; -fn sql_hook<'py>(entry: &Bound<'py, PyAny>, sql_fields: &[String]) -> PyResult> { +pub(crate) fn sql_hook<'py>( + entry: &Bound<'py, PyAny>, + sql_fields: &[String], +) -> PyResult> { let hook: Bound<'py, PyDict> = PyDict::new(entry.py()); hook.set_item("type", "sql")?; for field in sql_fields { @@ -37,51 +39,12 @@ fn sql_hook<'py>(entry: &Bound<'py, PyAny>, sql_fields: &[String]) -> PyResult( - value: &Bound<'py, PyAny>, - sql_fields: &[String], - python_hook_fields: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let py = value.py(); - let hooks: Bound<'py, PyList> = PyList::empty(py); - if !value.is_instance_of::() && !value.is_instance_of::() { - return Ok(hooks); - } - let sql_class: Bound<'py, PyAny> = record_class(py, RecordId::SqlHookEntry)?; - let python_class: Bound<'py, PyAny> = record_class(py, RecordId::PythonHookEntry)?; - for entry in value.try_iter()? { - let entry: Bound<'py, PyAny> = entry?; - if entry.is_instance(&sql_class)? { - hooks.append(sql_hook(&entry, sql_fields)?)?; - } else if entry.is_instance(&python_class)? { - let hook: Bound<'py, PyDict> = PyDict::new(py); - let name: Bound<'py, PyAny> = native_field(&entry, "name")?; - hook.set_item("type", "python")?; - hook.set_item("name", &name)?; - hook.set_item("kwargs", native_field(&entry, "kwargs")?)?; - if let Some(fields) = python_hook_fields - && let Some(extra) = fields.get_item(&name)? - { - hook.call_method1("update", (extra,))?; - } - hooks.append(hook)?; - } - } - Ok(hooks) -} - #[pyfunction] #[pyo3(signature = (*, value, sql_fields, python_hook_fields = None))] -fn serialize_hook_entries<'py>( +pub(crate) fn serialize_hook_entries<'py>( value: &Bound<'py, PyAny>, sql_fields: Vec, python_hook_fields: Option>, ) -> PyResult> { hook_entries(value, &sql_fields, python_hook_fields.as_ref()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(serialize_hook_entries, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/key_checks.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/key_checks.rs index a19f1a9e74..bc7b62f748 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/headers/key_checks.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/headers/key_checks.rs @@ -1,13 +1,12 @@ //! Unknown and unsupported authored keys, rejected with nearest-key suggestions natively. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::pyfunction; use sqlbuild_discovery::key_checks::main::unknown_mapping_keys::unknown_mapping_keys as unknown_keys; use sqlbuild_discovery::key_checks::main::unsupported_keys_help::unsupported_keys_help as keys_help; /// `(message after the label, help)` when a mapping key is outside `allowed`. #[pyfunction] -fn unknown_mapping_keys( +pub(crate) fn unknown_mapping_keys( keys: Vec<(String, bool)>, allowed: Vec, ) -> Option<(String, String)> { @@ -16,12 +15,7 @@ fn unknown_mapping_keys( /// `unsupported_keys_help` for `keys` outside `supported_keys`. #[pyfunction] -fn unsupported_keys_help(keys: Vec, supported_keys: Vec) -> String { +pub(crate) fn unsupported_keys_help(keys: Vec, supported_keys: Vec) -> String { let keys: Vec<&str> = keys.iter().map(String::as_str).collect(); keys_help(&keys, &supported_keys) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(unknown_mapping_keys, module)?)?; - module.add_function(wrap_pyfunction!(unsupported_keys_help, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/configuration.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/configuration.rs index 5e720d022e..7bd7806e38 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/configuration.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/configuration.rs @@ -1,71 +1,19 @@ //! Project inputs holding only the project and local configuration. +use crate::discovery::main::project::load_local_config_file::load_local_config_file; +use crate::discovery::main::project::load_project_config_file::load_project_config_file; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyString; use sqlbuild_host::entries::models::ExecutorHost; -use crate::compile_driver::main::reject_invalid_connection_configuration::reject_invalid_connection_configuration; -use crate::compile_driver::main::reject_invalid_project_configuration::reject_invalid_project_configuration; use crate::compile_driver::models::ProjectConfiguration; -use crate::discovery::_helpers::project::project_config::load_local_config_file; -use crate::discovery::_helpers::project::project_config::load_project_config_file; -use crate::discovery::_helpers::project::project_config::warn_deprecated_settings; -use crate::discovery::_helpers::python_files::imported_providers::discover_provider_classes; -use crate::discovery::models::SecretPolicy; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -/// `DiscoveredProjectInputs` with the loaded configuration, inside the project discovery lifecycle. -pub(crate) fn discover_project_configuration<'py>( - py: Python<'py>, - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - configuration_inputs(py, project_dir, |_| Ok(())) -} - -/// The validated configuration without a discovery lifecycle, which reading two files never needs. -pub(crate) fn discover_connection_configuration<'py>( - py: Python<'py>, - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - loaded_configuration(py, project_dir, reject_invalid_connection_configuration) -} - -/// The configuration with discovery's checks and warnings, and the provider classes, without importing any other project Python. -pub(crate) fn discover_configured_providers<'py>( - py: Python<'py>, - project_dir: &Bound<'py, PyAny>, - secret_policy: SecretPolicy, -) -> PyResult> { - let configuration: Bound<'py, PyAny> = loaded_configuration(py, project_dir, |config| { - reject_invalid_project_configuration(config)?; - warn_deprecated_settings(py, project_dir) - })?; - let providers: Bound<'py, PyAny> = - discover_provider_classes(project_dir.clone(), secret_policy.name())?; - new_record_with( - py, - RecordId::DiscoveredProjectInputs, - &[ - ("project_dir", project_dir.clone()), - ( - "project_config", - native_field(&configuration, "project_config")?, - ), - ( - "local_config", - native_field(&configuration, "local_config")?, - ), - ("providers", providers), - ], - ) -} - -fn configuration_inputs<'py>( +pub(crate) fn configuration_inputs<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, checks: impl FnOnce(&ProjectConfiguration<'_, '_>) -> PyResult<()>, @@ -86,7 +34,7 @@ fn configuration_inputs<'py>( } /// Both configuration files, after `checks` accepts them. -fn loaded_configuration<'py>( +pub(crate) fn loaded_configuration<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, checks: impl FnOnce(&ProjectConfiguration<'_, '_>) -> PyResult<()>, diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/locations.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/locations.rs index 8998d17785..faa1fea759 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/locations.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/locations.rs @@ -1,20 +1,10 @@ //! Authored locations Python consumers read: YAML entry lines and project-relative paths. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::{PyTuple, PyTupleMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; use sqlbuild_python_base::records::main::fields::field::field as native_field; -/// The 1-based line of the outermost `- name: ` entry, if present. -#[pyfunction] -#[pyo3(signature = (*, contents, name))] -pub(crate) fn yaml_entry_line(contents: &str, name: &str) -> Option { - yaml_entry_lines(contents) - .into_iter() - .find_map(|(entry, line)| (entry == name).then_some(line)) -} - /// `path.relative_to(project_dir)` without pathlib's per-parent comparison scan. #[pyfunction] #[pyo3(signature = (*, path, project_dir))] @@ -37,9 +27,3 @@ pub(crate) fn project_relative_path<'py>( path_parts.get_slice(root_parts.len(), path_parts.len()), ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(yaml_entry_line, module)?)?; - module.add_function(wrap_pyfunction!(project_relative_path, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs index 270be60e8e..2aaacb4b95 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs @@ -2,28 +2,15 @@ use std::path::PathBuf; +use pyo3::IntoPyObjectExt; use pyo3::exceptions::{PyOSError, PyValueError}; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_config::project::errors::{ConfigFailure, FailureClass, LoadFailure}; -use sqlbuild_config::project::main::deprecated_setting_warnings::deprecated_setting_warnings; -use sqlbuild_config::project::main::load_changes_config::load_changes_config; -use sqlbuild_config::project::main::load_git_base_ref::load_git_base_ref; -use sqlbuild_config::project::main::load_local_config::load_local_config; -use sqlbuild_config::project::main::load_project_config::load_project_config; -use sqlbuild_config::project::main::load_review_config::load_review_config; -use sqlbuild_config::project::main::local_sql_analysis_failure::local_sql_analysis_failure; -use sqlbuild_config::project::main::resolve_config_file::resolve_config_file; -use sqlbuild_config::project::models::{ - Built, ChangesConfig, ConfigFile, ConfigPaths, ReviewConfig, -}; +use sqlbuild_config::project::models::{Built, ConfigFile, ConfigPaths}; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_python_base::boundary::main::config_value_to_python::config_value_to_python; -use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; @@ -41,12 +28,12 @@ const PYTHON_ENUM_MODULES: [(&str, &str); 1] = [("AuditSeverity", "sqlbuild.compiler.auditing.types")]; /// The four configuration file paths of a project directory, as Python and native paths. -struct Files<'py> { - python: Vec>, - native: ConfigPaths, +pub(crate) struct Files<'py> { + pub(crate) python: Vec>, + pub(crate) native: ConfigPaths, } -fn files<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn files<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { let python = FILE_NAMES .iter() .map(|name| project_dir.call_method1("__truediv__", (*name,))) @@ -69,7 +56,7 @@ fn files<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { }) } -fn file_index(file: ConfigFile) -> usize { +pub(crate) fn file_index(file: ConfigFile) -> usize { match file { ConfigFile::Project => 0, ConfigFile::LegacyProject => 1, @@ -78,7 +65,11 @@ fn file_index(file: ConfigFile) -> usize { } } -fn config_error(py: Python<'_>, failure: &ConfigFailure, files: &Files<'_>) -> PyResult { +pub(crate) fn config_error( + py: Python<'_>, + failure: &ConfigFailure, + files: &Files<'_>, +) -> PyResult { use sqlbuild_config::project::errors::MessagePiece; let pieces = failure .message @@ -110,7 +101,11 @@ fn os_error(py: Python<'_>, error: &std::io::Error, path: &Bound<'_, PyAny>) -> Ok(PyErr::from_value(value)) } -fn load_error(py: Python<'_>, failure: &LoadFailure, files: &Files<'_>) -> PyResult { +pub(crate) fn load_error( + py: Python<'_>, + failure: &LoadFailure, + files: &Files<'_>, +) -> PyResult { match failure { LoadFailure::Config(failure) => config_error(py, failure, files), LoadFailure::Io(file, error) => os_error(py, error, &files.python[file_index(*file)]), @@ -142,7 +137,7 @@ fn object<'py>( new_record(py, record_named(class)?, (), Some(&keywords)) } -fn built(py: Python<'_>, value: Built) -> PyResult> { +pub(crate) fn built(py: Python<'_>, value: Built) -> PyResult> { match value { Built::Value(value) => config_value_to_python(py, value), Built::Object(class, fields) => Ok(object(py, class, fields)?.unbind()), @@ -168,101 +163,11 @@ fn built(py: Python<'_>, value: Built) -> PyResult> { } } -/// Write a warning line to stderr for each deprecated setting alias `project_dir` still sets. -pub(crate) fn warn_deprecated_settings( - py: Python<'_>, - project_dir: &Bound<'_, PyAny>, -) -> PyResult<()> { - let files = files(project_dir)?; - match deprecated_setting_warnings(&files.native) { - Ok(warnings) => warnings - .iter() - .try_for_each(|warning| write_console(py, "stderr", &format!("{warning}\n"))), - Err(failure) => Err(load_error(py, &failure, &files)?), - } -} - -/// `load_project_config`: the validated `ProjectConfig` of `project_dir`. -#[pyfunction] -pub(crate) fn load_project_config_file( - py: Python<'_>, - project_dir: &Bound<'_, PyAny>, -) -> PyResult> { - let files = files(project_dir)?; - match load_project_config(&files.native) { - Ok(config) => built(py, config), - Err(failure) => Err(load_error(py, &failure, &files)?), - } -} - -/// `[git] base_ref` of the project at `project_dir`, which `--since @base` names. -pub(crate) fn configured_git_base_ref(project_dir: &Bound<'_, PyAny>) -> PyResult> { - configured(project_dir, load_git_base_ref) -} - -/// `[review]` of the project at `project_dir`: the defaults `sqb review` falls back to. -pub(crate) fn configured_review(project_dir: &Bound<'_, PyAny>) -> PyResult { - configured(project_dir, load_review_config) -} - -/// `[changes]` of the project at `project_dir`: what `--changed` counts and reaches by default. -pub(crate) fn configured_changes(project_dir: &Bound<'_, PyAny>) -> PyResult { - configured(project_dir, load_changes_config) -} - /// One configuration section of the project at `project_dir`, with load failures raised as Python's. -fn configured( +pub(crate) fn configured( project_dir: &Bound<'_, PyAny>, load: fn(&ConfigPaths) -> Result, ) -> PyResult { let files: Files<'_> = files(project_dir)?; load(&files.native).or_else(|failure| Err(load_error(project_dir.py(), &failure, &files)?)) } - -/// `load_local_config`: the validated `LocalConfig` of `project_dir`, or the default. -#[pyfunction] -pub(crate) fn load_local_config_file( - py: Python<'_>, - project_dir: &Bound<'_, PyAny>, -) -> PyResult> { - let files = files(project_dir)?; - match load_local_config(&files.native) { - Ok(config) => built(py, config), - Err(failure) => Err(load_error(py, &failure, &files)?), - } -} - -/// The project or local configuration file path, rejecting an unconverted legacy file. -#[pyfunction] -pub(crate) fn resolve_config_file_path<'py>( - py: Python<'py>, - project_dir: &Bound<'py, PyAny>, - local: bool, -) -> PyResult> { - let files = files(project_dir)?; - match resolve_config_file(&files.native, local) { - Ok(file) => Ok(files.python[file_index(file)].clone()), - Err(failure) => Err(config_error(py, &failure, &files)?), - } -} - -/// `validate_local_sql_analysis_policy` from the project requirement and the local setting. -#[pyfunction] -pub(crate) fn validate_local_sql_analysis( - py: Python<'_>, - project_dir: &Bound<'_, PyAny>, - flags: (bool, bool, bool), -) -> Result<(), PyErr> { - let files = files(project_dir)?; - match local_sql_analysis_failure(&files.native, flags) { - None => Ok(()), - Some(failure) => Err(config_error(py, &failure, &files)?), - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(load_project_config_file, module)?)?; - module.add_function(wrap_pyfunction!(load_local_config_file, module)?)?; - module.add_function(wrap_pyfunction!(resolve_config_file_path, module)?)?; - module.add_function(wrap_pyfunction!(validate_local_sql_analysis, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs index 003e2a881e..1fdf94746a 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs @@ -1,26 +1,26 @@ //! The facts cross-file validation reads, taken from Python's `DiscoveredProjectInputs`. -use crate::discovery::models::ModelFileTexts; -use crate::discovery::models::NativeModelFile; +use crate::discovery::main::project::column_name::column_name; +use crate::discovery::main::project::seed_csv_settings::seed_csv_settings; +use crate::discovery::main::project::stem::stem; use crate::discovery::models::NodeRow; +use crate::discovery::models::{ModelFileTexts, NativeModelFile}; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTypeMethods}; use sqlbuild_discovery::project_validation::errors::CsvDialectError; use sqlbuild_discovery::project_validation::models::{ AuditFileFact, CsvSettings, DependencyFact, IntegrationLoaderFact, ModelFact, NamedPath, - NodeFact, NodeKind, ProjectFacts, SchemaFileFact, ScopedDeclarationsFact, SeedEntryFact, - SeedFileFact, SourceFact, SourceFileFact, TestBlockFact, TestFileFact, + NodeFact, NodeKind, SchemaFileFact, ScopedDeclarationsFact, SeedEntryFact, SeedFileFact, + SourceFact, SourceFileFact, TestBlockFact, TestFileFact, }; -use sqlbuild_python_base::compile_context::main::delta::injection::injection; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::path_stem::path_stem; const CSV_MODULE: &str = "csv"; /// `DeclarationKind.AUDIT`: a generic audit file. const GENERIC_AUDIT_KIND: &str = "audit"; /// Each Python node kind and the discovered inputs field that lists it, in discovery order. -const PYTHON_NODE_FIELDS: [(&str, &str); 4] = [ +pub(crate) const PYTHON_NODE_FIELDS: [(&str, &str); 4] = [ ("loader", "loader_functions"), ("task", "task_functions"), ("asset", "asset_functions"), @@ -32,65 +32,6 @@ const PRIVATE_SCOPE: &str = "private"; /// `(SQL resource name, owning node index, decorated name)` of one dependency. pub(crate) type DependencyRow = (Option, Option, Option); -/// The facts of `inputs`, with the Python nodes the facade has described. -pub(crate) fn project_facts( - py: Python<'_>, - inputs: &Bound<'_, PyAny>, - nodes: Vec, -) -> PyResult { - let mut declaration_files: Vec = - each(inputs, "enum_files", declarations_fact)?; - declaration_files.extend(each(inputs, "constant_files", declarations_fact)?); - let mut model_schemas: Vec = Vec::new(); - for schema in native_field(inputs, "model_schema_files")?.try_iter()? { - model_schemas.extend(each(&schema?, "declarations", named)?); - } - Ok(ProjectFacts { - models: each(inputs, "model_files", model_fact)?, - sql_functions: each(inputs, "sql_function_files", stem_named)?, - python_functions: each(inputs, "python_function_files", stem_named)?, - sql_hooks: each(inputs, "sql_hook_files", named)?, - seeds: each(inputs, "seed_files", seed_file_fact)?, - schemas: each(inputs, "schema_files", |schema| schema_fact(py, schema))?, - sources: each(inputs, "source_files", source_file_fact)?, - tests: each(inputs, "test_files", test_file_fact)?, - scenarios: each(inputs, "scenario_files", named)?, - audits: each(inputs, "audit_files", audit_fact)?, - declaration_files, - model_schemas, - materializations: each(inputs, "materialization_files", named)?, - nodes: nodes.into_iter().map(node_fact).collect::>()?, - hook_functions: each(inputs, "hook_functions", named)?, - event_exporters: each(inputs, "event_exporters", named)?, - providers: each(inputs, "providers", named)?, - path_default_keys: native_field(inputs, "project_config")? - .getattr("path_defaults")? - .try_iter()? - .map(|key| key?.extract()) - .collect::>()?, - project_model_paths: injection().map(|injection| injection.project_model_paths.clone()), - csv_field_limit: Some(csv_field_limit(py)?), - orchestration: Vec::new(), - }) -} - -/// The discovered Python nodes by kind, name and path only, as the resource-name check reads them. -pub(crate) fn named_node_rows(inputs: &Bound<'_, PyAny>) -> PyResult> { - let mut rows: Vec = Vec::new(); - for (kind, name) in PYTHON_NODE_FIELDS { - rows.extend(each(inputs, name, |node| { - Ok(NodeRow { - kind: kind.to_owned(), - name: native_field(node, "name")?.extract()?, - path: text(node, "relative_path")?, - decorator_config: false, - depends_on: Vec::new(), - }) - })?); - } - Ok(rows) -} - /// `read` applied to each item of the iterable attribute `name` of `owner`. pub(crate) fn each( owner: &Bound<'_, PyAny>, @@ -104,21 +45,16 @@ pub(crate) fn each( Ok(items) } -fn text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { +pub(crate) fn text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { Ok(native_field(owner, name)?.str()?.to_string()) } -fn optional_text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult> { +pub(crate) fn optional_text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult> { native_field(owner, name)?.extract() } -/// `str(Path(...).stem)` of the path attribute `name`. -pub(crate) fn stem(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { - path_stem(&native_field(owner, name)?) -} - /// `(resource.name, str(resource.relative_path))`. -fn named(resource: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn named(resource: &Bound<'_, PyAny>) -> PyResult { Ok(NamedPath { name: native_field(resource, "name")?.extract()?, path: text(resource, "relative_path")?, @@ -126,7 +62,7 @@ fn named(resource: &Bound<'_, PyAny>) -> PyResult { } /// `(file.file_path.stem, str(file.relative_path))`; a native model file answers from its texts. -fn stem_named(file: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn stem_named(file: &Bound<'_, PyAny>) -> PyResult { if let Ok(model_file) = file.cast::() { let texts: ModelFileTexts = model_file.get().path_texts(file.py())?; return Ok(NamedPath { @@ -152,7 +88,7 @@ fn enum_value(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { native_field(&native_field(owner, name)?, "value")?.extract() } -fn model_fact(model: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn model_fact(model: &Bound<'_, PyAny>) -> PyResult { let mut local_declarations: Vec = each(model, "enum_declarations", named)?; local_declarations.extend(each(model, "constant_declarations", named)?); Ok(ModelFact { @@ -161,30 +97,13 @@ fn model_fact(model: &Bound<'_, PyAny>) -> PyResult { }) } -fn seed_file_fact(seed: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn seed_file_fact(seed: &Bound<'_, PyAny>) -> PyResult { Ok(SeedFileFact { file: stem_named(seed)?, absolute_path: text(seed, "file_path")?, }) } -pub(crate) fn column_name(column: &Bound<'_, PyAny>) -> PyResult { - native_field(column, "name")?.extract() -} - -/// The CSV reader settings a seed declaration authors, without the dialect check. -pub(crate) fn seed_csv_settings(entry: &Bound<'_, PyAny>) -> PyResult { - let settings: Bound<'_, PyAny> = native_field(entry, "csv_settings")?; - Ok(CsvSettings { - delimiter: optional_text(&settings, "delimiter")?, - quotechar: optional_text(&settings, "quotechar")?, - escapechar: optional_text(&settings, "escapechar")?, - doublequote: native_field(&settings, "doublequote")?.extract()?, - skipinitialspace: native_field(&settings, "skipinitialspace")?.extract()?, - dialect_error: None, - }) -} - fn seed_entry_fact(py: Python<'_>, entry: &Bound<'_, PyAny>) -> PyResult { let mut csv: CsvSettings = seed_csv_settings(entry)?; csv.dialect_error = csv_dialect_error(py, &csv)?; @@ -195,7 +114,7 @@ fn seed_entry_fact(py: Python<'_>, entry: &Bound<'_, PyAny>) -> PyResult, schema: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn schema_fact(py: Python<'_>, schema: &Bound<'_, PyAny>) -> PyResult { Ok(SchemaFileFact { path: text(schema, "relative_path")?, model_names: each(schema, "model_entries", column_name)?, @@ -220,7 +139,7 @@ fn source_fact(entry: &Bound<'_, PyAny>) -> PyResult { }) } -fn source_file_fact(file: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn source_file_fact(file: &Bound<'_, PyAny>) -> PyResult { Ok(SourceFileFact { path: text(file, "relative_path")?, entries: each(file, "source_entries", source_fact)?, @@ -234,7 +153,7 @@ fn test_block_fact(block: &Bound<'_, PyAny>) -> PyResult { }) } -fn test_file_fact(file: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn test_file_fact(file: &Bound<'_, PyAny>) -> PyResult { Ok(TestFileFact { file: relative_stem_named(file)?, blocks: each(file, "blocks", test_block_fact)?, @@ -245,7 +164,7 @@ fn block_name(block: &Bound<'_, PyAny>) -> PyResult> { optional_text(block, "name") } -fn audit_fact(file: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn audit_fact(file: &Bound<'_, PyAny>) -> PyResult { Ok(AuditFileFact { file: relative_stem_named(file)?, generic: enum_value(file, "declaration_kind")? == GENERIC_AUDIT_KIND, @@ -253,7 +172,7 @@ fn audit_fact(file: &Bound<'_, PyAny>) -> PyResult { }) } -fn declarations_fact(file: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn declarations_fact(file: &Bound<'_, PyAny>) -> PyResult { Ok(ScopedDeclarationsFact { private: enum_value(file, "scope_kind")? == PRIVATE_SCOPE, declarations: each(file, "declarations", named)?, @@ -282,7 +201,7 @@ fn dependency_fact((sql_name, owner, decorated_name): DependencyRow) -> Dependen } } -fn node_fact(node: NodeRow) -> PyResult { +pub(crate) fn node_fact(node: NodeRow) -> PyResult { Ok(NodeFact { kind: node_kind(&node.kind)?, name: node.name, @@ -321,7 +240,7 @@ fn csv_dialect_error(py: Python<'_>, csv: &CsvSettings) -> PyResult) -> PyResult { +pub(crate) fn csv_field_limit(py: Python<'_>) -> PyResult { py.import(CSV_MODULE)? .call_method0("field_size_limit")? .extract() diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_files.rs index c60e8febc9..5d5e9b12ec 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_files.rs @@ -1,11 +1,9 @@ //! Native discovery results as the plain data the Python discovery facade materialises. use pyo3::exceptions::PyException; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyTuple; -use pyo3::{IntoPyObjectExt, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction, pymethods}; use sqlbuild_discovery::model_files::main::parse_model_text::parse_model_text; use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use sqlbuild_discovery::tree::models::ProjectTree; @@ -14,12 +12,10 @@ use std::path::Path; use crate::discovery::_helpers::failures::raised::failure_error; use crate::discovery::_helpers::headers::header_declarations::model_header_declarations; use crate::discovery::_helpers::records::declaration_records::source_locations; -use crate::discovery::_helpers::records::model_file_record::DiscoveredPaths; -use crate::discovery::_helpers::records::model_file_record::ModelFileParts; -use crate::discovery::_helpers::records::model_file_record::PendingModelFile; -use crate::discovery::_helpers::snapshots::native_classes::ModelTextRequest; -use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; -use crate::discovery::models::NativeModelFile; +use crate::discovery::_helpers::records::model_file_record::{ + DiscoveredPaths, ModelFileParts, PendingModelFile, +}; +use crate::discovery::_helpers::snapshots::native_classes::{ModelTextRequest, NativeProjectTree}; use crate::sqltext::main::map_to_python::map_to_python; use crate::sqltext::main::optional_authored_value_to_python::optional_authored_value_to_python; use sqlbuild_python_base::types::CompilerDetach; @@ -96,7 +92,7 @@ pub(crate) fn model_record( /// Parse model contents as discovery does: `(header_values, query_sql, column locations)`. #[pyfunction] -fn parse_model_contents( +pub(crate) fn parse_model_contents( py: Python<'_>, request: ModelTextRequest, contents: String, @@ -124,11 +120,3 @@ fn parse_model_contents( .unbind(), )) } - -/// Whether native discovery reproduces the string semantics of this Python and Unicode version. -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_function(wrap_pyfunction!(parse_model_contents, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs index 9086c6bdb8..7b412f439f 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs @@ -1,27 +1,23 @@ //! Cross-file validation of one discovery's inputs. -use pyo3::exceptions::PyKeyError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::discovery::main::project::project_facts::project_facts; +use crate::discovery::main::project::validation_error::validation_error; +use crate::discovery::main::sources::orchestration_declarations::orchestration_declarations; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_discovery::project_validation::main::validate_project_inputs::validate_project_inputs; -use sqlbuild_discovery::project_validation::models::{ProjectFacts, ValidationFailure}; +use sqlbuild_discovery::project_validation::models::ProjectFacts; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::read_error; -use crate::discovery::_helpers::project::project_facts::project_facts; -use crate::discovery::_helpers::sources::orchestration_files::orchestration_declarations; use crate::discovery::models::NodeRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; -const CSV_MODULE: &str = "csv"; -const BUILTINS_MODULE: &str = "builtins"; -const PATHLIB_MODULE: &str = "pathlib"; +pub(crate) const CSV_MODULE: &str = "csv"; +pub(crate) const BUILTINS_MODULE: &str = "builtins"; +pub(crate) const PATHLIB_MODULE: &str = "pathlib"; /// The exceptions a CSV dialect may raise, by class name. -const CSV_DIALECT_ERRORS: [&str; 2] = ["TypeError", "ValueError"]; +pub(crate) const CSV_DIALECT_ERRORS: [&str; 2] = ["TypeError", "ValueError"]; -fn raised(result: PyResult>) -> PyErr { +pub(crate) fn raised(result: PyResult>) -> PyErr { match result { Ok(error) => PyErr::from_value(error), Err(error) => error, @@ -30,7 +26,7 @@ fn raised(result: PyResult>) -> PyErr { /// Raise the first cross-file conflict across the discovered inputs, as Python raises it. #[pyfunction] -fn validate_discovered_project_inputs( +pub(crate) fn validate_discovered_project_inputs( py: Python<'_>, inputs: Bound<'_, PyAny>, nodes: Vec, @@ -42,43 +38,3 @@ fn validate_discovered_project_inputs( } validate_project_inputs(&facts).map_err(|failure| validation_error(py, failure)) } - -/// The Python exception a failed project validation raises. -pub(crate) fn validation_error(py: Python<'_>, failure: ValidationFailure) -> PyErr { - match failure { - ValidationFailure::Discovery(failure) => failure_error(py, failure), - ValidationFailure::Unreadable { - absolute_path, - failure, - } => match py - .import(PATHLIB_MODULE) - .and_then(|module| native_field(&module, "Path")) - .and_then(|path| path.call1((absolute_path,))) - { - Ok(file_path) => read_error(failure, &file_path), - Err(error) => error, - }, - ValidationFailure::Csv(message) => raised( - py.import(CSV_MODULE) - .and_then(|module| native_field(&module, "Error")) - .and_then(|class| class.call1((message,))), - ), - ValidationFailure::CsvDialect(error) => { - if !CSV_DIALECT_ERRORS.contains(&error.class.as_str()) { - return PyKeyError::new_err(error.class.to_string()); - } - raised( - py.import(BUILTINS_MODULE) - .and_then(|module| native_field(&module, error.class.as_str())) - .and_then(|class| class.call1((error.message,))), - ) - } - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - validate_discovered_project_inputs, - module - )?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs index 7e04bf4870..0fd34a7d4c 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs @@ -2,7 +2,7 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; +use pyo3::types::{PyTuple, PyTupleMethods}; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use sqlbuild_host::entries::models::UserCodeHost; @@ -132,30 +132,6 @@ pub(crate) fn is_exception(py: Python<'_>, error: &PyErr) -> bool { error.is_instance_of::(py) } -/// `{provider.name: provider for provider in providers}`. -pub(crate) fn provider_by_name<'py>(providers: &Bound<'py, PyAny>) -> PyResult> { - let by_name: Bound<'py, PyDict> = PyDict::new(providers.py()); - for provider in providers.try_iter()? { - let provider: Bound<'py, PyAny> = provider?; - by_name.set_item(native_field(&provider, "name")?, provider)?; - } - Ok(by_name) -} - -/// The provider parameters of one loaded user function. -pub(crate) fn provider_usages<'py>( - function: &Bound<'py, PyAny>, - provider_by_name: &Bound<'py, PyDict>, -) -> PyResult> { - call_with( - &shim(function.py(), UserCodeHost::ProviderUsages)?, - &[ - ("function", function.clone()), - ("provider_by_name", provider_by_name.clone().into_any()), - ], - ) -} - /// A tuple of the given items. pub(crate) fn tuple_of<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_providers.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_providers.rs index c592f4f592..1fea680c08 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_providers.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_providers.rs @@ -1,5 +1,7 @@ //! Provider classes and lifecycle-event sinks from imported providers/ and sinks/ modules. +use crate::discovery::main::python_files::provider_by_name::provider_by_name; +use crate::discovery::main::python_files::provider_usages::provider_usages; use std::collections::HashSet; use pyo3::exceptions::PyRuntimeError; @@ -9,15 +11,9 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyType}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::discovery::_helpers::python_files::imported_modules::ProjectDir; -use crate::discovery::_helpers::python_files::imported_modules::discovered; -use crate::discovery::_helpers::python_files::imported_modules::discovery_error; -use crate::discovery::_helpers::python_files::imported_modules::discovery_error_from; -use crate::discovery::_helpers::python_files::imported_modules::module_functions; -use crate::discovery::_helpers::python_files::imported_modules::provider_by_name; -use crate::discovery::_helpers::python_files::imported_modules::provider_usages; -use crate::discovery::_helpers::python_files::imported_modules::shim; -use crate::discovery::_helpers::python_files::imported_modules::tuple_of; +use crate::discovery::_helpers::python_files::imported_modules::{ + ProjectDir, discovered, discovery_error, discovery_error_from, module_functions, shim, tuple_of, +}; use crate::discovery::constants::ALLOW_MISSING_SECRETS_FLAG; use crate::discovery::models::SecretPolicy; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -45,7 +41,7 @@ const DEFAULT_ERROR_TYPE: &str = "validation_error"; const DEFAULT_SETTINGS_ERROR: &str = "invalid provider settings"; const RECALL_WRITER_UNINSTALLED: &str = "the native module registered no RecallWriter class"; -static RECALL_WRITER: PyOnceLock> = PyOnceLock::new(); +pub(crate) static RECALL_WRITER: PyOnceLock> = PyOnceLock::new(); fn reject_invalid_identity( name: &Bound<'_, PyAny>, @@ -362,11 +358,6 @@ pub(crate) fn discover_event_exporter_declarations<'py>( tuple_of(py, declarations) } -/// Install the `RecallWriter` class sink parameters are recognised by, once, at registration. -pub(crate) fn install_recall_writer(writer: &Bound<'_, PyType>) { - let _ = RECALL_WRITER.get_or_init(writer.py(), || writer.clone().unbind()); -} - /// Each parameter of `function` with its annotation, and whether that annotation is `RecallWriter`. fn annotated_parameters<'py>( function: &Bound<'py, PyAny>, @@ -388,7 +379,7 @@ fn annotated_parameters<'py>( } /// Whether `declaration`'s function takes a `RecallWriter`, making it a Recall replacement sink. -fn takes_recall_writer(declaration: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn takes_recall_writer(declaration: &Bound<'_, PyAny>) -> PyResult { Ok( annotated_parameters(&native_field(declaration, "function")?)? .iter() @@ -469,31 +460,3 @@ pub(crate) fn bind_event_exporter_declarations<'py>( } tuple_of(py, bound) } - -/// Validated lifecycle-event sink declarations under sinks/, bound to providers; Recall replacement sinks are left out. -#[pyfunction] -#[pyo3(signature = (*, project_dir, providers = None))] -pub(crate) fn discover_event_exporter_functions<'py>( - project_dir: Bound<'py, PyAny>, - providers: Option>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let project = ProjectDir { - path: project_dir.clone(), - }; - if let Some(cached) = project.cached_extensions()? { - return cached.get_item(1); - } - let mut ordinary: Vec> = Vec::new(); - for declaration in discover_event_exporter_declarations(project_dir.clone())?.try_iter()? { - let declaration: Bound<'py, PyAny> = declaration?; - if !takes_recall_writer(&declaration)? { - ordinary.push(declaration); - } - } - bind_event_exporter_declarations( - tuple_of(py, ordinary)?, - providers.map_or_else(|| tuple_of(py, Vec::new()), Ok)?, - project_dir, - ) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs index 68221f5848..c9f981204b 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs @@ -1,29 +1,19 @@ //! Decorated Python nodes, audit factories, hooks and materializations from imported modules. -use sqlbuild_python_base::native_host::main::within_identity_pass::within_identity_pass; +use crate::discovery::main::python_files::provider_by_name::provider_by_name; +use crate::discovery::main::python_files::provider_usages::provider_usages; use std::collections::HashMap; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyList, PyString, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_core::constants::PYTHON_NODE_ROOT; use sqlbuild_host::entries::models::UserCodeHost; -use crate::discovery::_helpers::python_files::imported_modules::ProjectDir; -use crate::discovery::_helpers::python_files::imported_modules::definition; -use crate::discovery::_helpers::python_files::imported_modules::discovered; -use crate::discovery::_helpers::python_files::imported_modules::discovery_error; -use crate::discovery::_helpers::python_files::imported_modules::discovery_error_from; -use crate::discovery::_helpers::python_files::imported_modules::is_exception; -use crate::discovery::_helpers::python_files::imported_modules::module_functions; -use crate::discovery::_helpers::python_files::imported_modules::provider_by_name; -use crate::discovery::_helpers::python_files::imported_modules::provider_usages; -use crate::discovery::_helpers::python_files::imported_modules::shim; -use crate::discovery::_helpers::python_files::imported_modules::tuple_of; -use crate::discovery::_helpers::python_files::imported_providers::bind_event_exporter_declarations; -use crate::discovery::_helpers::python_files::imported_providers::discover_event_exporter_declarations; -use crate::discovery::_helpers::python_files::imported_providers::discover_event_exporter_functions; -use crate::discovery::_helpers::python_files::imported_providers::discover_provider_classes; +use crate::discovery::_helpers::python_files::imported_modules::{ + ProjectDir, definition, discovered, discovery_error, discovery_error_from, is_exception, + module_functions, shim, tuple_of, +}; use crate::discovery::_helpers::python_files::python_hook_files::discover_python_hook_files; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -113,9 +103,9 @@ const NODE_KINDS: [NodeKind; 4] = [ ]; /// Discovered loaders, tasks, assets, checks and audit factories, in discovery order. -struct Buckets<'py> { - nodes: [Vec>; 4], - audit_factories: Vec<(String, String, Bound<'py, PyAny>)>, +pub(crate) struct Buckets<'py> { + pub(crate) nodes: [Vec>; 4], + pub(crate) audit_factories: Vec<(String, String, Bound<'py, PyAny>)>, } /// The module and the file it was imported from. @@ -261,7 +251,7 @@ impl<'py> Buckets<'py> { Ok(()) } - fn discover_nodes( + pub(crate) fn discover_nodes( &mut self, project: &ProjectDir<'py>, providers: &Bound<'py, PyDict>, @@ -406,64 +396,6 @@ fn factory_functions<'py>( Ok(functions) } -/// Decorated Python DAG node functions under python/. -#[pyfunction] -#[pyo3(signature = (*, project_dir, providers = None))] -pub(crate) fn discover_python_node_functions<'py>( - project_dir: Bound<'py, PyAny>, - providers: Option>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let project = ProjectDir { path: project_dir }; - let providers: Bound<'py, PyDict> = match providers { - Some(providers) => provider_by_name(&providers)?, - None => PyDict::new(py), - }; - let mut buckets = Buckets { - nodes: [Vec::new(), Vec::new(), Vec::new(), Vec::new()], - audit_factories: Vec::new(), - }; - let node_root: Bound<'py, PyAny> = project.child(PYTHON_NODE_ROOT)?; - if node_root.call_method0("is_dir")?.is_truthy()? { - let saved: Bound<'py, PyAny> = call_with( - &shim(py, UserCodeHost::InstallPythonRootPackage)?, - &[ - ("node_root", node_root), - ("project_dir", project.path.clone()), - ], - )?; - let discovered_nodes: PyResult<()> = - within_identity_pass(py, || buckets.discover_nodes(&project, &providers)); - call_with( - &shim(py, UserCodeHost::RestorePythonRootModules)?, - &[("saved_modules", saved)], - )?; - discovered_nodes?; - } - let [loaders, tasks, assets, checks] = buckets.nodes; - discovered( - py, - RecordId::DiscoveredPythonNodeFunctions, - &[ - ("loaders", tuple_of(py, loaders)?), - ("tasks", tuple_of(py, tasks)?), - ("assets", tuple_of(py, assets)?), - ("checks", tuple_of(py, checks)?), - ( - "audit_factories", - tuple_of( - py, - buckets - .audit_factories - .into_iter() - .map(|(_, _, record)| record) - .collect(), - )?, - ), - ], - ) -} - /// Decorated model lifecycle hook functions under hooks/python/. #[pyfunction] #[pyo3(signature = (*, project_dir, providers = None))] @@ -524,62 +456,6 @@ pub(crate) fn discover_hook_functions<'py>( tuple_of(py, hooks) } -/// Custom materialization Python files under materializations/. -#[pyfunction] -#[pyo3(signature = (*, project_dir, providers = None))] -pub(crate) fn discover_materialization_files<'py>( - project_dir: Bound<'py, PyAny>, - providers: Option>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let project = ProjectDir { path: project_dir }; - if !project - .child("materializations")? - .call_method0("is_dir")? - .is_truthy()? - { - return tuple_of(py, Vec::new()); - } - let providers: Bound<'py, PyDict> = match providers { - Some(providers) => provider_by_name(&providers)?, - None => PyDict::new(py), - }; - let mut files: Vec> = Vec::new(); - for file_path in project.module_files("materializations", false)? { - let materialize: Bound<'py, PyAny> = - project.load(UserCodeHost::LoadMaterializeFunction, &file_path)?; - let usages: Bound<'py, PyAny> = if materialize.is_none() { - tuple_of(py, Vec::new())? - } else { - provider_usages(&materialize, &providers)? - }; - files.push(discovered( - py, - RecordId::DiscoveredMaterializationFile, - &[ - ("file_path", file_path.clone()), - ("relative_path", project.relative(&file_path)?), - ("name", native_field(&file_path, "stem")?), - ("provider_usages", usages), - ], - )?); - } - tuple_of(py, files) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_python_node_functions, module)?)?; - module.add_function(wrap_pyfunction!(discover_hook_functions, module)?)?; - module.add_function(wrap_pyfunction!(discover_materialization_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_provider_classes, module)?)?; - module.add_function(wrap_pyfunction!( - discover_event_exporter_declarations, - module - )?)?; - module.add_function(wrap_pyfunction!(bind_event_exporter_declarations, module)?)?; - module.add_function(wrap_pyfunction!(discover_event_exporter_functions, module)?) -} - fn optional_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { if value.is_none() { return Ok(value.clone()); diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs index 68aa0b7878..42d3474bf8 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs @@ -1,30 +1,18 @@ //! Python function files discovered natively, with CPython as the authority on syntax and values. use pyo3::exceptions::{PySyntaxError, PyTypeError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{ - PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, -}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_discovery::declaration_files::main::python_function_files::python_function_files; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods}; use sqlbuild_discovery::key_checks::main::unsupported_keys_help::unsupported_keys_help; -use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome, ProjectRoot}; +use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome}; use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_pysource::udf_functions::constants::UDF_KEYS; use sqlbuild_pysource::udf_functions::main::udf_function::udf_function; use sqlbuild_pysource::udf_functions::models::{UdfFunction, UdfShape}; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::read_error; -use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::failures::raised::{failure_error, read_error}; use crate::discovery::_helpers::project::project_files::path_object; use crate::discovery::_helpers::records::declaration_records::path_class; -use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; -use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; -use crate::discovery::_helpers::snapshots::snapshot::project_root; -use crate::discovery::_helpers::snapshots::snapshot::project_tree; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -235,7 +223,7 @@ fn parsed_function<'py>( Ok((values, entry_point, warehouse_body(&module, &function)?)) } -fn function_record<'py>( +pub(crate) fn function_record<'py>( project_dir: &Bound<'py, PyAny>, tree: &ProjectTree, file: DiscoveredFile, @@ -268,29 +256,3 @@ fn function_record<'py>( ], ) } - -/// Discover the `@udf(...)` Python function files under `functions/python/`. -#[pyfunction] -#[pyo3(signature = (*, project_dir))] -pub(crate) fn discover_python_function_files<'py>( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - in_snapshot(project_dir, || { - let py: Python<'py> = project_dir.py(); - let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; - let root: ProjectRoot = project_root(project_dir)?; - let inner: &ProjectTree = &tree.get().inner; - let files: Vec> = py - .detach(|| python_function_files(&root, inner)) - .map_err(|failure| stage_error(inner, project_dir, failure))?; - let records: Vec> = files - .into_iter() - .map(|file| function_record(project_dir, inner, file)) - .collect::>()?; - PyTuple::new(py, records) - }) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_python_function_files, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_hook_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_hook_files.rs index ded9531391..b7988b6cf0 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_hook_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_hook_files.rs @@ -1,24 +1,23 @@ //! The Python hook files of every hook role, listed natively for the Python module loader. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::declaration_files::main::python_hook_files::python_hook_files; use sqlbuild_discovery::declaration_files::models::{ CollectionFailure, DiscoverySession, FileScope, }; use sqlbuild_discovery::tree::models::ProjectTree; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::failures::raised::{failure_error, stage_error}; use crate::discovery::_helpers::project::project_files::path_object; -use crate::discovery::_helpers::records::declaration_records::add_scope_fields; -use crate::discovery::_helpers::records::declaration_records::discovery_session; -use crate::discovery::_helpers::records::declaration_records::path_class; -use crate::discovery::_helpers::snapshots::native_classes::NativeDiscoverySession; -use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; -use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; -use crate::discovery::_helpers::snapshots::snapshot::project_tree; +use crate::discovery::_helpers::records::declaration_records::{ + add_scope_fields, discovery_session, path_class, +}; +use crate::discovery::_helpers::snapshots::native_classes::{ + NativeDiscoverySession, NativeProjectTree, +}; +use crate::discovery::_helpers::snapshots::snapshot::{in_snapshot, project_tree}; /// Each public Python hook file as `file_path`, `relative_path` and its role's scope fields. #[pyfunction] @@ -53,7 +52,3 @@ pub(crate) fn discover_python_hook_files<'py>( PyTuple::new(py, files) }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_python_hook_files, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_module_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_module_files.rs index 2afbf6ce18..c0bb6592c5 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_module_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_module_files.rs @@ -1,8 +1,8 @@ //! The project Python module files Python imports, listed natively in Python's sorted order. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyTuple; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::declaration_files::main::python_module_files::python_module_files; use sqlbuild_discovery::tree::models::ProjectTree; @@ -31,7 +31,3 @@ pub(crate) fn discover_python_module_files<'py>( .collect::>()?; PyTuple::new(py, paths) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_python_module_files, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_paths.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_paths.rs index 07873c43ed..001e70123c 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_paths.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_paths.rs @@ -1,32 +1,4 @@ //! Reject project Python outside documented executable roots. -use std::path::PathBuf; - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_discovery::python_paths::main::unsupported_python_paths::unsupported_python_paths; - -use crate::discovery::_helpers::failures::raised::discovery_error; - -const PYTHON_PATH_HELP: &str = "Move pipeline Python nodes and the helper modules they import under \ +pub(crate) const PYTHON_PATH_HELP: &str = "Move pipeline Python nodes and the helper modules they import under \ python/, or move SQLBuild extensions under their documented extension path."; - -/// Raise `ProjectPythonPathError` naming every unsupported Python file below `project_dir`. -pub(crate) fn reject_unsupported_python_paths(project_dir: &Bound<'_, PyAny>) -> PyResult<()> { - let root: PathBuf = project_dir.extract()?; - let unsupported: Vec = unsupported_python_paths(&root) - .into_iter() - .map(|parts| parts.join(std::path::MAIN_SEPARATOR_STR)) - .collect(); - if unsupported.is_empty() { - return Ok(()); - } - Err(discovery_error( - project_dir.py(), - "python_path", - format!( - "Unsupported project Python path(s): {}", - unsupported.join(", ") - ), - Some(PYTHON_PATH_HELP.to_owned()), - )) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs index 0e0bb863b9..e1c0c868e3 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs @@ -1,8 +1,8 @@ //! Declaration file collections discovered natively and built as Python discovery records. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_discovery::declaration_files::main::discover_declaration_collection::discover_declaration_collection; use sqlbuild_discovery::declaration_files::main::retained_collection::retained_collection; use sqlbuild_discovery::declaration_files::models::{ @@ -17,19 +17,16 @@ use sqlbuild_host::entries::models::FacadeHost; use sqlbuild_sqltext::compiler::models::AuthoredValue; use std::sync::Arc; -use crate::discovery::_helpers::failures::faults::FileFaults; -use crate::discovery::_helpers::failures::faults::record_tuple; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::read_error; -use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::failures::faults::{FileFaults, record_tuple}; +use crate::discovery::_helpers::failures::raised::{failure_error, read_error, stage_error}; use crate::discovery::_helpers::headers::header_declarations::typed_constant_record; use crate::discovery::_helpers::project::project_files::path_object; -use crate::discovery::_helpers::snapshots::native_classes::NativeDiscoverySession; -use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; -use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; -use crate::discovery::_helpers::snapshots::snapshot::project_root; -use crate::discovery::_helpers::snapshots::snapshot::project_tree; -use crate::discovery::_helpers::snapshots::snapshot::snapshot_memo; +use crate::discovery::_helpers::snapshots::native_classes::{ + NativeDiscoverySession, NativeProjectTree, +}; +use crate::discovery::_helpers::snapshots::snapshot::{ + in_snapshot, project_root, project_tree, snapshot_memo, +}; use crate::model_config::main::headers::schema_columns::schema_columns; use crate::sqltext::main::map_to_python::map_to_python; use sqlbuild_discovery::declaration_files::main::declaration_file_options::declaration_file_options; @@ -43,7 +40,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; const PATHLIB_MODULE: &str = "pathlib"; -const SESSION_MEMO_KEY: &str = "native_discovery_session"; +pub(crate) const SESSION_MEMO_KEY: &str = "native_discovery_session"; const MEASUREMENT_MODE: &str = "MEASUREMENT"; /// One declaration file being built: its paths and the scope native discovery listed it in. @@ -391,19 +388,6 @@ pub(crate) fn discovery_session<'py>( Ok(created) } -/// The session this discovery pass read its declaration files with, if any. -#[pyfunction] -#[pyo3(signature = (*, project_dir))] -pub(crate) fn retained_discovery_session<'py>( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - Ok(snapshot_memo(project_dir)? - .get_item(SESSION_MEMO_KEY)? - .filter(|session| session.is_instance_of::()) - .unwrap_or_else(|| py.None().into_bound(py))) -} - /// One collection read in the pass's snapshot, with the tree it was read from. fn collection<'py>( project_dir: &Bound<'py, PyAny>, @@ -487,7 +471,7 @@ fn records<'py, T>( record_tuple(py, built) } -fn discovered<'py>( +pub(crate) fn discovered<'py>( faults: &FileFaults<'py>, kind: CollectionKind, isolate_kind: bool, @@ -538,7 +522,7 @@ fn seed_records<'py, T>( record_tuple(py, seeds) } -fn faults<'py>( +pub(crate) fn faults<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, ) -> FileFaults<'py> { @@ -547,147 +531,3 @@ fn faults<'py>( on_fault: on_fault.filter(|on_fault| !on_fault.is_none()), } } - -/// Discover global and scoped enum declaration files. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None, isolate_declaration_kind=false))] -pub(crate) fn discover_enum_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, - isolate_declaration_kind: bool, -) -> PyResult> { - discovered( - &faults(project_dir, on_fault), - CollectionKind::Enums, - isolate_declaration_kind, - ) -} - -/// Discover global and scoped constant declaration files. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None, isolate_declaration_kind=false))] -pub(crate) fn discover_constant_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, - isolate_declaration_kind: bool, -) -> PyResult> { - discovered( - &faults(project_dir, on_fault), - CollectionKind::Constants, - isolate_declaration_kind, - ) -} - -/// Discover project-wide and scoped reusable model schemas. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None))] -pub(crate) fn discover_model_schema_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, -) -> PyResult> { - discovered( - &faults(project_dir, on_fault), - CollectionKind::ModelSchemas, - false, - ) -} - -/// Discover SQL function files under functions/sql/. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None))] -pub(crate) fn discover_sql_function_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, -) -> PyResult> { - discovered( - &faults(project_dir, on_fault), - CollectionKind::SqlFunctions, - false, - ) -} - -/// Discover named SQL lifecycle hooks in their project-wide and scoped roles. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None))] -pub(crate) fn discover_sql_hook_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, -) -> PyResult> { - discovered( - &faults(project_dir, on_fault), - CollectionKind::SqlHooks, - false, - ) -} - -/// Discover generic and singular audit SQL files in their project-wide and scoped roles. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None))] -pub(crate) fn discover_audit_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, -) -> PyResult> { - discovered( - &faults(project_dir, on_fault), - CollectionKind::Audits, - false, - ) -} - -/// Discover global and scoped project macro Python files. -#[pyfunction] -#[pyo3(signature = (*, project_dir, isolate_declaration_kind=false))] -pub(crate) fn discover_macro_files<'py>( - project_dir: &Bound<'py, PyAny>, - isolate_declaration_kind: bool, -) -> PyResult> { - discovered( - &faults(project_dir, None), - CollectionKind::Macros, - isolate_declaration_kind, - ) -} - -/// Discover seed CSV files under seeds/. -#[pyfunction] -#[pyo3(signature = (*, project_dir))] -pub(crate) fn discover_seed_files<'py>( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - discovered(&faults(project_dir, None), CollectionKind::Seeds, false) -} - -/// Detect a project-level adapter.py without importing it. -#[pyfunction] -#[pyo3(signature = (*, project_dir))] -pub(crate) fn discover_adapter_file<'py>( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let file_path: Bound<'py, PyAny> = project_dir.div("adapter.py")?; - if !file_path.call_method0("is_file")?.is_truthy()? { - return Ok(py.None().into_bound(py)); - } - new_record_with( - py, - RecordId::DiscoveredAdapterFile, - &[ - ("file_path", file_path), - ("relative_path", path_class(py)?.call1(("adapter.py",))?), - ], - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_enum_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_constant_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_model_schema_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_sql_function_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_sql_hook_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_audit_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_macro_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_seed_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_adapter_file, module)?)?; - module.add_function(wrap_pyfunction!(retained_discovery_session, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/description_inputs.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/description_inputs.rs index 8789e2ac0e..76942ba545 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/records/description_inputs.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/description_inputs.rs @@ -1,16 +1,14 @@ //! Model description resolution for header-only tooling, without importing project Python. -use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyCFunction, PyDict, PyFrozenSet}; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::discovery::main::project::load_project_config_file::load_project_config_file; +use crate::discovery::main::records::discover_model_schema_files::discover_model_schema_files; +use crate::discovery::main::records::discovered_model_files::discovered_model_files; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyCFunction; -use crate::discovery::_helpers::project::project_config::load_project_config_file; -use crate::discovery::_helpers::records::declaration_records::discover_model_schema_files; -use crate::discovery::_helpers::records::model_records::discovered_model_files; use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; use crate::discovery::models::ModelFileSelection; -use crate::model_config::main::model_nodes::externally_described_models::externally_described_models as described_models; -use crate::render::main::declarations::public_model_schema_index::public_model_schema_index; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::types::RecordId; @@ -59,18 +57,3 @@ pub(crate) fn discover_model_description_inputs<'py>( ) }) } - -/// Model files that `[defaults]`, `[path_defaults]` or a model schema describe. -pub(crate) fn externally_described_models<'py>( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let inputs: Bound<'py, PyAny> = discover_model_description_inputs(project_dir)?; - let schemas: Bound<'py, PyDict> = public_model_schema_index(inputs.clone())?; - PyFrozenSet::new(py, described_models(inputs, schemas)?) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_model_description_inputs, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_file_record.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_file_record.rs index 7c0b3caf9b..213afb1855 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_file_record.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_file_record.rs @@ -205,18 +205,6 @@ fn posix(text: &str) -> String { text.replace(std::path::MAIN_SEPARATOR, "/") } -/// `text` as the Python string `cache` shares with every reader, built on first read. -pub(crate) fn projected_text<'py>( - py: Python<'py>, - cache: &PyOnceLock>, - text: &str, -) -> Bound<'py, PyString> { - cache - .get_or_init(py, || PyString::new(py, text).unbind()) - .bind(py) - .clone() -} - /// A model header's authored values: native entries, with the Python dict built on first read. pub(crate) struct HeaderValues { pub(crate) native: Option>, diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_records.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_records.rs index 606e49bb74..5ddaf7edf0 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/model_records.rs @@ -1,46 +1,36 @@ //! SQL model files discovered natively and built as Python discovery records. -use crate::discovery::models::ModelFileSelection; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; use sqlbuild_discovery::model_files::main::discover_model_files::discover_model_files as discover_models; use sqlbuild_discovery::model_files::main::model_file_stem::model_file_stem; use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome, StageFailure}; use std::path::Path; -use crate::discovery::_helpers::failures::faults::FileFaults; -use crate::discovery::_helpers::failures::faults::record_tuple; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::read_error; -use crate::discovery::_helpers::failures::raised::stage_error; -use crate::discovery::_helpers::project::project_files::model_record; -use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::failures::raised::{failure_error, read_error, stage_error}; +use crate::discovery::_helpers::project::project_files::{model_record, path_object}; use crate::discovery::_helpers::records::model_file_record::DiscoveredPaths; use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; -use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; -use crate::discovery::_helpers::snapshots::snapshot::project_root; -use crate::discovery::_helpers::snapshots::snapshot::project_tree; -use sqlbuild_discovery::model_files::main::model_file_options::model_file_options; +use crate::discovery::_helpers::snapshots::snapshot::{in_snapshot, project_root, project_tree}; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; const PATHLIB_MODULE: &str = "pathlib"; const STEM_ATTRIBUTE: &str = "stem"; /// One discovered model file: its relative path and record, or the error building it raises. -type ModelItem<'py> = (RelativePath<'py>, PyResult>); +pub(crate) type ModelItem<'py> = (RelativePath<'py>, PyResult>); /// A discovered file's relative path: native text, or the Python path already built for it. -enum RelativePath<'py> { +pub(crate) enum RelativePath<'py> { Native(String), Python(Bound<'py, PyAny>), } impl<'py> RelativePath<'py> { - fn object(&self, py: Python<'py>) -> PyResult> { + pub(crate) fn object(&self, py: Python<'py>) -> PyResult> { match self { Self::Native(relative) => Path::new(relative).into_pyobject(py), Self::Python(relative) => Ok(relative.clone()), @@ -48,7 +38,7 @@ impl<'py> RelativePath<'py> { } /// The model's name, its file stem. - fn name(&self, py: Python<'py>) -> PyResult> { + pub(crate) fn name(&self, py: Python<'py>) -> PyResult> { match self { Self::Native(relative) => Ok(PyString::new(py, model_file_stem(relative)).into_any()), Self::Python(relative) => native_field(relative, STEM_ATTRIBUTE), @@ -57,7 +47,7 @@ impl<'py> RelativePath<'py> { } /// Discover every model file natively, each as its record or the error it raises when built. -fn model_items<'py>( +pub(crate) fn model_items<'py>( project_dir: &Bound<'py, PyAny>, options: &ModelFileOptions, ) -> PyResult>> { @@ -106,81 +96,3 @@ fn model_items<'py>( Ok(items) }) } - -/// The (selected) model files; `extraction` is `(implicit alias columns, output locations)`. -pub(crate) fn discovered_model_files<'py>( - project_dir: &Bound<'py, PyAny>, - extraction: (bool, bool), - selection: ModelFileSelection<'py>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let options: ModelFileOptions = model_file_options( - extraction, - sqlbuild_python_base::compile_context::main::delta::injection::injection() - .map(|injection| std::sync::Arc::new(injection.model_paths.clone())), - ); - let items: Vec> = model_items(project_dir, &options)?; - let faults = FileFaults { - project_dir: project_dir.clone(), - on_fault: selection.on_fault.filter(|on_fault| !on_fault.is_none()), - }; - let selected: Option> = selection - .selected_model_names - .filter(|selected| !selected.is_none()); - let mut records: Vec> = Vec::with_capacity(items.len()); - for (relative_path, built) in items { - if let Some(selected) = &selected - && !selected.contains(relative_path.name(py)?)? - { - continue; - } - match built { - Ok(record) => records.push(record), - Err(error) => { - records.extend(faults.kept(&relative_path.object(py)?, Err(error))?); - } - } - } - record_tuple(py, records) -} - -/// Discover the (selected) model files natively, reporting failing files to `on_fault`. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn discover_model_files<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "discover_model_files", - options, - ( - &[ - "project_dir", - "extract_implicit_alias_columns", - "extract_output_column_locations", - ], - &["selected_model_names", "on_fault"], - ), - )?; - discovered_model_files( - &arguments.value("project_dir")?, - ( - arguments - .value("extract_implicit_alias_columns")? - .extract()?, - arguments - .value("extract_output_column_locations")? - .extract()?, - ), - ModelFileSelection { - selected_model_names: Some(arguments.value("selected_model_names")?), - on_fault: Some(arguments.value("on_fault")?), - }, - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_model_files, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs index 1dd3ed910b..4c0bf51b14 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs @@ -1,34 +1,27 @@ //! SQL unit-test and scenario files discovered natively and built as Python discovery records. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{ PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, }; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_discovery::models::{DiscoveredFile, DiscoveryFailure, FileOutcome, StageFailure}; -use sqlbuild_discovery::sql_tests::main::discover_scenario_files::discover_scenario_files as discover_scenarios; -use sqlbuild_discovery::sql_tests::main::discover_sql_test_files::discover_sql_test_files as discover_tests; use sqlbuild_discovery::sql_tests::main::parse_scenario_text::parse_scenario_text; -use sqlbuild_discovery::sql_tests::main::parse_sql_test_text::parse_sql_test_text; use sqlbuild_discovery::sql_tests::models::{ DiscoveredScenarioFile, DiscoveredSqlTestFile, SqlTestBlock, SqlTestFileOptions, }; use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_model_config::execution_limits::main::parse_execution_limit::parse_execution_limit; -use crate::discovery::_helpers::failures::faults::FileFaults; -use crate::discovery::_helpers::failures::faults::record_tuple; -use crate::discovery::_helpers::failures::raised::discovery_error; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::read_error; -use crate::discovery::_helpers::failures::raised::stage_error; -use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::failures::faults::{FileFaults, record_tuple}; +use crate::discovery::_helpers::failures::raised::{ + discovery_error, failure_error, read_error, stage_error, +}; use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; -use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; -use crate::discovery::_helpers::snapshots::snapshot::project_root; -use crate::discovery::_helpers::snapshots::snapshot::project_tree; -use crate::discovery::_helpers::sources::sql_test_blocks::sql_test_block_records; -use crate::discovery::_helpers::sources::sql_test_blocks::validate_block_names; +use crate::discovery::_helpers::snapshots::snapshot::{in_snapshot, project_root, project_tree}; +use crate::discovery::_helpers::sources::sql_test_blocks::{ + sql_test_block_records, validate_block_names, +}; use crate::sqltext::main::map_to_python::map_to_python; use sqlbuild_discovery::constants; use sqlbuild_discovery::sql_tests::main::sql_test_file_options::sql_test_file_options; @@ -56,14 +49,14 @@ type Discoverer = fn( type RecordBuilder<'py, T> = fn((&Bound<'py, PyAny>, &Bound<'py, PyAny>), T) -> PyResult>; -fn options() -> PyResult { +pub(crate) fn options() -> PyResult { Ok(sql_test_file_options( sqlbuild_python_base::compile_context::main::delta::injection::injection() .map(|injection| std::sync::Arc::new(injection.test_paths.clone())), )) } -fn path_class(py: Python<'_>) -> PyResult> { +pub(crate) fn path_class(py: Python<'_>) -> PyResult> { imported(py, PATHLIB_MODULE, "Path") } @@ -73,7 +66,7 @@ fn ownership_root<'py>(py: Python<'py>, root: &str) -> PyResult( +pub(crate) fn discovered_files<'py, T: Send>( project_dir: &Bound<'py, PyAny>, discover: Discoverer, ) -> PyResult<(Bound<'py, NativeProjectTree>, Vec>)> { @@ -93,7 +86,7 @@ fn discovered_files<'py, T: Send>( } /// Each block as a `DiscoveredSqlTestBlock`, then the split failure, then the name check. -fn test_blocks<'py>( +pub(crate) fn test_blocks<'py>( file_path: &Bound<'py, PyAny>, blocks: Vec, failure: Option, @@ -127,7 +120,7 @@ fn test_blocks<'py>( PyTuple::new(py, records) } -fn test_file<'py>( +pub(crate) fn test_file<'py>( paths: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), file: DiscoveredSqlTestFile, ) -> PyResult> { @@ -153,7 +146,7 @@ fn test_file<'py>( } /// Python's `build_sql_scenario_file`: validate the header and build the record. -fn scenario_file<'py>( +pub(crate) fn scenario_file<'py>( paths: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), file: DiscoveredScenarioFile, ) -> PyResult> { @@ -229,7 +222,7 @@ fn scenario_file<'py>( } /// Build each file, raising or reporting each failing one. -fn records<'py, T>( +pub(crate) fn records<'py, T>( faults: &FileFaults<'py>, files: Vec<(Bound<'py, PyAny>, FileOutcome)>, build: RecordBuilder<'py, T>, @@ -248,81 +241,6 @@ fn records<'py, T>( record_tuple(py, built) } -/// Discover the (selected) SQL test files natively, reporting failing files to `on_fault`. -#[pyfunction] -#[pyo3(signature = (*, project_dir, selected_paths=None, on_fault=None))] -pub(crate) fn discover_test_files<'py>( - project_dir: &Bound<'py, PyAny>, - selected_paths: Option>, - on_fault: Option>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let (tree, files) = discovered_files(project_dir, discover_tests)?; - let tree: &ProjectTree = &tree.get().inner; - let mut kept: Vec<(Bound<'py, PyAny>, FileOutcome)> = Vec::new(); - for file in files { - let relative: Bound<'py, PyAny> = path_object(py, tree, file.relative_path)?.into_bound(py); - if let Some(selected) = &selected_paths - && !selected.contains(project_dir.div(&relative)?.call_method0("resolve")?)? - { - continue; - } - kept.push((path_class(py)?.call1((relative,))?, file.outcome)); - } - records( - &FileFaults { - project_dir: project_dir.clone(), - on_fault, - }, - kept, - test_file, - ) -} - -/// Discover scenario files natively, reporting failing files to `on_fault`. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None))] -pub(crate) fn discover_scenario_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let (tree, files) = discovered_files(project_dir, discover_scenarios)?; - let tree: &ProjectTree = &tree.get().inner; - let kept: Vec<(Bound<'py, PyAny>, FileOutcome)> = files - .into_iter() - .map(|file| { - Ok(( - path_class(py)?.call1((path_object(py, tree, file.relative_path)?,))?, - file.outcome, - )) - }) - .collect::>()?; - records( - &FileFaults { - project_dir: project_dir.clone(), - on_fault, - }, - kept, - scenario_file, - ) -} - -/// Parse in-memory SQL test contents into ordered blocks, as test discovery does. -#[pyfunction] -#[pyo3(signature = (*, contents, file_path))] -pub(crate) fn parse_sql_test_file<'py>( - contents: String, - file_path: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = file_path.py(); - let path: String = file_path.str()?.to_str()?.to_owned(); - let options: SqlTestFileOptions = options()?; - let parsed = py.detach(|| parse_sql_test_text(&path, contents, &options)); - let file: DiscoveredSqlTestFile = parsed.map_err(|failure| failure_error(py, failure))?; - test_blocks(file_path, file.blocks, file.failure) -} - /// Parse in-memory scenario contents into the record scenario discovery builds for a file. #[pyfunction] #[pyo3(signature = (*, contents, file_path, relative_path))] @@ -338,10 +256,3 @@ pub(crate) fn parse_sql_scenario_file<'py>( let file: DiscoveredScenarioFile = parsed.map_err(|failure| failure_error(py, failure))?; scenario_file((file_path, relative_path), file) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_test_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_scenario_files, module)?)?; - module.add_function(wrap_pyfunction!(parse_sql_test_file, module)?)?; - module.add_function(wrap_pyfunction!(parse_sql_scenario_file, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs index 964e001533..2a70e191a3 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs @@ -1,51 +1,39 @@ //! Source and schema YAML files discovered natively and built as Python discovery records. +use crate::discovery::main::snapshots::escaped::escaped; use pyo3::exceptions::PyUnicodeEncodeError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyList, PyStringMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyList, PyStringMethods}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_config::models::ConfigValue; use sqlbuild_discovery::models::{DiscoveryFailure, FailureKind, ProjectRoot, StageFailure}; use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_discovery::yaml_files::main::load_yaml_files::load_yaml_files as load_yaml; use sqlbuild_discovery::yaml_files::main::load_yaml_text::load_yaml_text; -use sqlbuild_discovery::yaml_files::main::schema_yaml_paths::schema_yaml_paths; -use sqlbuild_discovery::yaml_files::main::source_yaml_paths::source_yaml_paths; -use sqlbuild_discovery::yaml_files::models::{YamlFileOutcome, YamlPathFailure}; +use sqlbuild_discovery::yaml_files::models::YamlFileOutcome; use sqlbuild_python_base::compile_context::main::yaml_cache::retain_read_sources::retain_read_sources; use std::path::PathBuf; -use crate::discovery::_helpers::failures::faults::FileFaults; -use crate::discovery::_helpers::failures::faults::record_tuple; -use crate::discovery::_helpers::failures::raised::discovery_error; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::read_error; -use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::failures::raised::{ + discovery_error, failure_error, read_error, stage_error, +}; use crate::discovery::_helpers::project::project_files::path_object; -use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; -use crate::discovery::_helpers::snapshots::native_classes::YamlDiscoveryRequest; -use crate::discovery::_helpers::snapshots::snapshot::escaped; -use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; -use crate::discovery::_helpers::snapshots::snapshot::project_root; -use crate::discovery::_helpers::snapshots::snapshot::project_tree; -use crate::discovery::_helpers::sources::yaml_entries::parse_schema_document; -use crate::discovery::_helpers::sources::yaml_entries::parse_sources_document; +use crate::discovery::_helpers::snapshots::native_classes::{ + NativeProjectTree, YamlDiscoveryRequest, +}; +use crate::discovery::_helpers::snapshots::snapshot::{project_root, project_tree}; use sqlbuild_python_base::boundary::main::config_value_to_python::config_value_to_python; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::types::RecordId; /// The most YAML bytes one native load reads before its files are built and released. const YAML_BATCH_BYTES: u64 = 4 * 1024 * 1024; const PATHLIB_MODULE: &str = "pathlib"; const PROJECT_PATH_KIND: &str = "project_path"; -const SCHEMA_KIND: &str = "schema"; +pub(crate) const SCHEMA_KIND: &str = "schema"; /// The YAML files one discovery reads, and the kind their failures raise as. #[derive(Clone, Copy)] -enum YamlKind { +pub(crate) enum YamlKind { Source, Schema, } @@ -59,12 +47,12 @@ impl YamlKind { } } -fn path_class(py: Python<'_>) -> PyResult> { +pub(crate) fn path_class(py: Python<'_>) -> PyResult> { imported(py, PATHLIB_MODULE, "Path") } /// `(contents, loaded)` of a loaded file; a value Python cannot build fails like invalid YAML. -fn loaded<'py>( +pub(crate) fn loaded<'py>( py: Python<'py>, (display_path, kind): (&str, FailureKind), (contents, value): (String, ConfigValue), @@ -101,7 +89,7 @@ fn outcome_loaded<'py>( } /// The pass's tree, or a private one for an undecodable project directory (fails on first read). -fn listing_tree<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn listing_tree<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { match project_dir.str()?.to_str() { Ok(_) => Ok(ListingTree::Shared(project_tree(project_dir)?)), Err(error) if error.is_instance_of::(project_dir.py()) => { @@ -112,13 +100,13 @@ fn listing_tree<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult { +pub(crate) enum ListingTree<'py> { Shared(Bound<'py, NativeProjectTree>), Private(ProjectTree), } impl ListingTree<'_> { - fn tree(&self) -> &ProjectTree { + pub(crate) fn tree(&self) -> &ProjectTree { match self { Self::Shared(tree) => &tree.get().inner, Self::Private(tree) => tree, @@ -148,7 +136,7 @@ fn batches(tree: &ProjectTree, paths: Vec, limit: u64) -> Vec( +pub(crate) fn load_files<'py>( project_dir: &Bound<'py, PyAny>, (paths, kind): (Vec, YamlKind), mut on_file: impl FnMut( @@ -233,113 +221,6 @@ fn unnamed_error(project_dir: &Bound<'_, PyAny>, relative: &Bound<'_, PyAny>) -> )) } -/// Discover source YAML files natively, reporting failing files to `on_fault`. -#[pyfunction] -#[pyo3(signature = (*, project_dir, on_fault=None))] -pub(crate) fn discover_source_files<'py>( - project_dir: &Bound<'py, PyAny>, - on_fault: Option>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - in_snapshot(project_dir, || { - let listing: ListingTree<'py> = listing_tree(project_dir)?; - let paths: Vec = source_yaml_paths(listing.tree()) - .map_err(|failure| stage_error(listing.tree(), project_dir, failure))?; - if paths.is_empty() { - return record_tuple(py, Vec::new()); - } - let faults = FileFaults { - project_dir: project_dir.clone(), - on_fault, - }; - let mut records: Vec> = Vec::new(); - load_files( - project_dir, - (paths, YamlKind::Source), - |(relative_path, file_path), built| { - let record = built.and_then(|(contents, loaded)| { - let shown: String = file_path.str()?.to_str()?.to_owned(); - new_record_with( - py, - RecordId::DiscoveredSourceFile, - &[ - ("file_path", file_path.clone()), - ("relative_path", relative_path.clone()), - ("contents", contents.into_bound_py_any(py)?), - ( - "source_entries", - parse_sources_document(py, loaded, &shown)?.into_any(), - ), - ], - ) - }); - records.extend(faults.kept(&relative_path, record)?); - Ok(()) - }, - )?; - record_tuple(py, records) - }) -} - -/// Discover model schema.yml and seed declaration files natively. -#[pyfunction] -#[pyo3(signature = (*, project_dir))] -pub(crate) fn discover_schema_files<'py>( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - in_snapshot(project_dir, || { - let listing: ListingTree<'py> = listing_tree(project_dir)?; - let paths: Vec = match schema_yaml_paths(listing.tree()) { - Ok(paths) => paths, - Err(YamlPathFailure::Stage(failure)) => { - return Err(stage_error(listing.tree(), project_dir, failure)); - } - Err(YamlPathFailure::YamlSeedDeclaration(path)) => { - let relative: Bound<'py, PyAny> = - path_class(py)?.call1((path_object(py, listing.tree(), path)?,))?; - return Err(discovery_error( - py, - SCHEMA_KIND, - format!( - "Seed declaration file {} must use .yml; .yaml is not supported", - relative.str()? - ), - None, - )); - } - }; - if paths.is_empty() { - return record_tuple(py, Vec::new()); - } - let mut records: Vec> = Vec::new(); - load_files( - project_dir, - (paths, YamlKind::Schema), - |(relative_path, file_path), built| { - let (contents, loaded) = built?; - let shown: String = file_path.str()?.to_str()?.to_owned(); - records.push(new_record_with( - py, - RecordId::DiscoveredSchemaFile, - &[ - ("file_path", file_path.clone()), - ("relative_path", relative_path), - ("contents", contents.into_bound_py_any(py)?), - ("model_entries", PyTuple::empty(py).into_any()), - ( - "seed_entries", - parse_schema_document(py, loaded, &shown)?.into_any(), - ), - ], - )?); - Ok(()) - }, - )?; - record_tuple(py, records) - }) -} - fn yaml_kind(kind: &str) -> PyResult { match kind { "source" => Ok(FailureKind::Source), @@ -352,7 +233,7 @@ fn yaml_kind(kind: &str) -> PyResult { /// Read and load YAML files: each file's `(contents, loaded)`, or the error it raises. #[pyfunction] -fn load_yaml_files<'py>( +pub(crate) fn load_yaml_files<'py>( py: Python<'py>, request: YamlDiscoveryRequest, relative_paths: Vec, @@ -382,7 +263,7 @@ fn load_yaml_files<'py>( /// Load one in-memory YAML document as a discovered `kind` file named `file_path` loads. #[pyfunction] -fn load_yaml_document<'py>( +pub(crate) fn load_yaml_document<'py>( py: Python<'py>, file_path: &str, text: String, @@ -393,10 +274,3 @@ fn load_yaml_document<'py>( load_yaml_text(file_path, &text, kind).map_err(|failure| failure_error(py, failure))?; Ok(loaded(py, (file_path, kind), (text, value))?.1) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_source_files, module)?)?; - module.add_function(wrap_pyfunction!(discover_schema_files, module)?)?; - module.add_function(wrap_pyfunction!(load_yaml_files, module)?)?; - module.add_function(wrap_pyfunction!(load_yaml_document, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs index ccffda6237..42e194af63 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs @@ -1,27 +1,23 @@ //! Tolerant discovery of the scope-relevant authored roots, keeping each independent file fault. +use crate::discovery::main::project::load_local_config_file::load_local_config_file; +use crate::discovery::main::project::load_project_config_file::load_project_config_file; +use crate::discovery::main::records::discover_audit_files::discover_audit_files; +use crate::discovery::main::records::discover_constant_files::discover_constant_files; +use crate::discovery::main::records::discover_enum_files::discover_enum_files; +use crate::discovery::main::records::discover_macro_files::discover_macro_files; +use crate::discovery::main::records::discover_model_schema_files::discover_model_schema_files; +use crate::discovery::main::records::discover_scenario_files::discover_scenario_files; +use crate::discovery::main::records::discover_source_files::discover_source_files; +use crate::discovery::main::records::discover_sql_function_files::discover_sql_function_files; +use crate::discovery::main::records::discover_sql_hook_files::discover_sql_hook_files; +use crate::discovery::main::records::discover_test_files::discover_test_files; +use crate::discovery::main::records::discovered_model_files::discovered_model_files; use pyo3::exceptions::{PyOSError, PySyntaxError, PyUnicodeError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use crate::discovery::_helpers::project::project_config::load_local_config_file; -use crate::discovery::_helpers::project::project_config::load_project_config_file; -use crate::discovery::_helpers::records::declaration_records::discover_audit_files; -use crate::discovery::_helpers::records::declaration_records::discover_constant_files; -use crate::discovery::_helpers::records::declaration_records::discover_enum_files; -use crate::discovery::_helpers::records::declaration_records::discover_macro_files; -use crate::discovery::_helpers::records::declaration_records::discover_model_schema_files; -use crate::discovery::_helpers::records::declaration_records::discover_sql_function_files; -use crate::discovery::_helpers::records::declaration_records::discover_sql_hook_files; use crate::discovery::_helpers::records::declaration_records::path_class; -use crate::discovery::_helpers::records::model_records::discovered_model_files; -use crate::discovery::_helpers::records::sql_test_records::discover_scenario_files; -use crate::discovery::_helpers::records::sql_test_records::discover_test_files; -use crate::discovery::_helpers::records::yaml_records::discover_source_files; -use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; use crate::discovery::models::ModelFileSelection; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -177,7 +173,9 @@ fn chained<'py>(py: Python<'py>, lists: &[&Bound<'py, PyList>]) -> PyResult(project_dir: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn tolerant_discovery<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = project_dir.py(); let faults = Faults { project_dir: project_dir.clone(), @@ -263,16 +261,3 @@ fn tolerant_discovery<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - in_snapshot(project_dir, || tolerant_discovery(project_dir)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_scope_snapshot, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs index a8d137b27b..5cd57480ef 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs @@ -1,19 +1,19 @@ //! The directory snapshot one discovery pass shares, and the native tree it memoises. -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRefMut, PyResult, Python, -}; +use crate::discovery::main::snapshots::escaped::escaped; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyRefMut, PyResult, Python}; +use pyo3::pymethods; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; -use pyo3::{pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_discovery::models::ProjectRoot; use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_python_base::boundary::main::context_variable::context_variable; use std::path::PathBuf; use crate::discovery::_helpers::failures::raised::discovery_error; -use crate::discovery::_helpers::snapshots::native_classes::NativeDirectorySnapshot; -use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; +use crate::discovery::_helpers::snapshots::native_classes::{ + NativeDirectorySnapshot, NativeProjectTree, +}; use crate::discovery::models::NativeDirectorySnapshotScope; const SNAPSHOT_VARIABLE: &str = "sqlbuild_directory_snapshot"; @@ -23,8 +23,8 @@ const PROJECT_PATH_KIND: &str = "project_path"; const SURROGATE_ESCAPE_BASE: u32 = 0xDC00; const ESCAPED_BYTE_FIRST: u16 = 0xDC80; const ESCAPED_BYTE_LAST: u16 = 0xDCFF; -const UTF16_ENCODING: &str = "utf-16-le"; -const SURROGATE_PASS: &str = "surrogatepass"; +pub(crate) const UTF16_ENCODING: &str = "utf-16-le"; +pub(crate) const SURROGATE_PASS: &str = "surrogatepass"; static ACTIVE_SNAPSHOT: PyOnceLock> = PyOnceLock::new(); @@ -140,25 +140,6 @@ impl NativeDirectorySnapshotScope { } } -/// Share one directory snapshot across every discovery call for `project_dir` inside the scope. -#[pyfunction] -#[pyo3(signature = (*, project_dir))] -pub(crate) fn directory_snapshot_scope( - project_dir: Bound<'_, PyAny>, -) -> NativeDirectorySnapshotScope { - NativeDirectorySnapshotScope { - project_dir: project_dir.unbind(), - token: None, - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_function(wrap_pyfunction!(directory_snapshot_scope, module)?)?; - Ok(()) -} - /// `str(project_dir / "_")` without its probe: the text printed before a relative path. pub(crate) fn display_prefix(project_dir: &Bound<'_, PyAny>) -> PyResult { let probed: String = project_dir.div(DISPLAY_PROBE)?.str()?.to_str()?.to_owned(); @@ -213,25 +194,7 @@ pub(crate) fn project_tree<'py>( Ok(created) } -/// Python's `_escaped`: undecodable name bytes as `\xNN`, other lone surrogates as `\uXXXX`. -pub(crate) fn escaped(text: &Bound<'_, PyString>) -> PyResult { - let encoded: Vec = text - .call_method1("encode", (UTF16_ENCODING, SURROGATE_PASS))? - .extract()?; - let units = encoded - .as_chunks::<2>() - .0 - .iter() - .map(|pair| u16::from_le_bytes(*pair)); - Ok(char::decode_utf16(units) - .map(|decoded| match decoded { - Ok(character) => character.to_string(), - Err(error) => escaped_unit(error.unpaired_surrogate()), - }) - .collect()) -} - -fn escaped_unit(unit: u16) -> String { +pub(crate) fn escaped_unit(unit: u16) -> String { if cfg!(unix) && (ESCAPED_BYTE_FIRST..=ESCAPED_BYTE_LAST).contains(&unit) { return format!("\\x{:02x}", u32::from(unit) - SURROGATE_ESCAPE_BASE); } diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/declaration_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/declaration_files.rs index 029e093f42..0b91dea1bc 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/declaration_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/declaration_files.rs @@ -1,8 +1,8 @@ //! In-memory declaration file parsing, for the oracle comparing native and Python parsing. use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_discovery::declaration_files::main::parse_declaration_text::parse_declaration_text; use sqlbuild_discovery::declaration_files::models::{ CollectionKind, DeclarationFileOptions, ParsedDeclarationText, @@ -10,7 +10,6 @@ use sqlbuild_discovery::declaration_files::models::{ use sqlbuild_discovery::models::FileOutcome; use crate::discovery::_helpers::snapshots::native_classes::DeclarationTextRequest; -use crate::discovery::_helpers::snapshots::native_classes::NativeDiscoverySession; use sqlbuild_python_base::types::CompilerDetach; const PARSED_TAG: &str = "ok"; @@ -19,7 +18,7 @@ const UNREADABLE_TAG: &str = "read"; /// Whether a `kind` declaration file with these contents parses (`ok`), fails or is unreadable. #[pyfunction] -fn parse_declaration_contents( +pub(crate) fn parse_declaration_contents( py: Python<'_>, request: DeclarationTextRequest, names: (String, String), @@ -54,9 +53,3 @@ fn parse_declaration_contents( ))), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(parse_declaration_contents, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_files.rs index 43e1733cdd..086922ecf7 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_files.rs @@ -1,35 +1,14 @@ //! Orchestration declarations read natively, raised as the Python exceptions discovery raises. -use std::path::PathBuf; - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use sqlbuild_discovery::orchestration_files::main::orchestration_inventory::orchestration_inventory; -use sqlbuild_discovery::orchestration_files::models::{ - OrchestrationFailure, OrchestrationInventory, -}; +use pyo3::prelude::{Bound, PyAny, PyErr}; +use sqlbuild_discovery::orchestration_files::models::OrchestrationFailure; use sqlbuild_discovery::tree::models::ProjectTree; -use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::failures::raised::relative_read_error; -use crate::discovery::_helpers::failures::raised::stage_error; -use crate::discovery::_helpers::sources::orchestration_values::check_orchestration_values; - -/// Every job, job group, calendar and sensor of `project_dir`, or `OrchestrationDeclarationError` (D017). -pub(crate) fn orchestration_declarations( - project_dir: &Bound<'_, PyAny>, -) -> PyResult { - let py: Python<'_> = project_dir.py(); - let root: PathBuf = project_dir.extract()?; - let tree = ProjectTree::new(&root); - let version = py.version_info(); - let inventory = py.detach(|| orchestration_inventory(&tree, (version.major, version.minor))); - let inventory = - inventory.map_err(|failure| orchestration_error(&tree, project_dir, failure))?; - check_orchestration_values(py, &inventory).map_err(|failure| failure_error(py, failure))?; - Ok(inventory) -} +use crate::discovery::_helpers::failures::raised::{ + failure_error, relative_read_error, stage_error, +}; -fn orchestration_error( +pub(crate) fn orchestration_error( tree: &ProjectTree, project_dir: &Bound<'_, PyAny>, failure: OrchestrationFailure, diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/provider_secrets.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/provider_secrets.rs index 2b83ef870b..334ad93016 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/provider_secrets.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/provider_secrets.rs @@ -10,30 +10,17 @@ use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::ErrorId; /// Discovered collections of sinks that run with every command. -const SINK_COLLECTIONS: [&str; 2] = ["event_exporters", "command_output_sinks"]; +pub(crate) const SINK_COLLECTIONS: [&str; 2] = ["event_exporters", "command_output_sinks"]; thread_local! { /// Whether the invocation running on this thread passed `--allow-missing-secrets`. pub(crate) static INVOCATION_ALLOWS_MISSING_SECRETS: Cell = const { Cell::new(false) }; } -/// Whether the running invocation passed `--allow-missing-secrets`. -pub(crate) fn invocation_allows_missing_secrets() -> bool { - INVOCATION_ALLOWS_MISSING_SECRETS.get() -} - -/// `(provider, consumer)` for each usage, named by the usage's own `consumer_name` or else by `consumer`. -pub(crate) fn provider_uses( - usages: &Bound<'_, PyAny>, +pub(crate) fn usage_pair( + usage: &Bound<'_, PyAny>, consumer: Option<&str>, -) -> PyResult> { - usages - .try_iter()? - .map(|usage| usage_pair(&usage?, consumer)) - .collect() -} - -fn usage_pair(usage: &Bound<'_, PyAny>, consumer: Option<&str>) -> PyResult<(String, String)> { +) -> PyResult<(String, String)> { let provider: String = field(usage, "provider_name")?.extract()?; let name: String = match consumer { Some(name) => name.to_owned(), @@ -42,31 +29,6 @@ fn usage_pair(usage: &Bound<'_, PyAny>, consumer: Option<&str>) -> PyResult<(Str Ok((provider, name)) } -/// `(provider, consumer)` for each provider the named consumers, such as checks or loaders, inject. -pub(crate) fn consumer_uses(consumers: &Bound<'_, PyAny>) -> PyResult> { - let mut uses: Vec<(String, String)> = Vec::new(); - for consumer in consumers.try_iter()? { - let consumer: Bound<'_, PyAny> = consumer?; - let name: String = field(&consumer, "name")?.extract()?; - uses.extend(provider_uses( - &field(&consumer, "provider_usages")?, - Some(&name), - )?); - } - Ok(uses) -} - -/// The providers the project's lifecycle and command-output sinks use, which run with every command. -pub(crate) fn sink_uses(discovered: &Bound<'_, PyAny>) -> PyResult> { - let mut uses: Vec<(String, String)> = Vec::new(); - for collection in SINK_COLLECTIONS { - if let Some(sinks) = discovered.getattr_opt(collection)? { - uses.extend(consumer_uses(&sinks)?); - } - } - Ok(uses) -} - /// Fail before anything runs when a provider in `uses` lacks required secrets, listing each with what needs it. pub(crate) fn require_used_secrets( discovered: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs index 74a8d6921f..cdfa999a41 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs @@ -1,10 +1,8 @@ //! Loaded schema and source YAML documents parsed natively into the Python contract objects. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{FromPyObject, pyfunction}; use sqlbuild_discovery::yaml_entries::main::parse_schema_document::parse_schema_document as parse_schema; use sqlbuild_discovery::yaml_entries::main::parse_sources_document::parse_sources_document as parse_sources; use sqlbuild_discovery::yaml_entries::models::{ @@ -383,8 +381,3 @@ fn loaded_sources(sources: &[ParsedSourceEntry>]) -> Vec { }) .collect() } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_schema_document, module)?)?; - module.add_function(wrap_pyfunction!(parse_sources_document, module)?) -} diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs index 9fbbcfeb55..811c4e2f4d 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/explicit_references_help.rs @@ -1,9 +1,10 @@ //! Explicit references help. -use crate::discovery::_helpers::failures::guidance; +use pyo3::pyfunction; +use sqlbuild_config::project::main::explicit_references_help::explicit_references_help as references_help; -/// Explicit references help. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, allowed))] pub fn explicit_references_help(allowed: &str) -> String { - guidance::explicit_references_help(allowed) + references_help(allowed) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs index 6b6a9694f2..15aab07517 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/microbatch_guidance.rs @@ -1,9 +1,9 @@ //! Microbatch guidance. -use crate::discovery::_helpers::failures::guidance; +use pyo3::pyfunction; +use sqlbuild_config::project::main::microbatch_guidance::microbatch_guidance as concurrency_guidance; -/// Microbatch guidance. -#[inline] +#[pyfunction] pub fn microbatch_guidance() -> (String, String) { - guidance::microbatch_guidance() + concurrency_guidance() } diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs index 1de4ee6af6..451a179aa5 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs @@ -1,12 +1,19 @@ //! Register the guidance functions and classes. -use crate::discovery::_helpers::failures::guidance; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::guidance::snippet_help; +use crate::discovery::_helpers::failures::guidance::sql_analysis_off_guidance; +use crate::discovery::main::failures::explicit_references_help::explicit_references_help; +use crate::discovery::main::failures::microbatch_guidance::microbatch_guidance; +use crate::discovery::main::failures::setting_help::setting_help; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the guidance functions and classes. -#[inline] pub fn register_guidance(module: &Bound<'_, PyModule>) -> PyResult<()> { - guidance::register(module) + module.add_function(wrap_pyfunction!(setting_help, module)?)?; + module.add_function(wrap_pyfunction!(snippet_help, module)?)?; + module.add_function(wrap_pyfunction!(sql_analysis_off_guidance, module)?)?; + module.add_function(wrap_pyfunction!(microbatch_guidance, module)?)?; + module.add_function(wrap_pyfunction!(explicit_references_help, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/register_raised.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/register_raised.rs index 923c3390b6..86bc484252 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/failures/register_raised.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/register_raised.rs @@ -1,12 +1,10 @@ //! Register the raised functions and classes. -use crate::discovery::_helpers::failures::raised; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::raised::_oracle_read_error; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the raised functions and classes. -#[inline] pub fn register_raised(module: &Bound<'_, PyModule>) -> PyResult<()> { - raised::register(module) + module.add_function(wrap_pyfunction!(_oracle_read_error, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/setting_help.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/setting_help.rs index e63a8de597..d52e8dfd6d 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/failures/setting_help.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/setting_help.rs @@ -1,13 +1,33 @@ //! Render `, set this in :` followed by the exact TOML section and key. -use crate::discovery::_helpers::failures::guidance; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::discovery::_helpers::failures::guidance::setting_text; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_config::project::main::setting_help_text::setting_help_text; +use sqlbuild_config::project::models::SettingSite; +use sqlbuild_python_base::boundary::models::KeywordArguments; /// Render `, set this in :` followed by the exact TOML section and key. -#[inline] +#[pyfunction] +#[pyo3(signature = (**options))] pub fn setting_help<'py>(py: Python<'py>, options: Option>) -> PyResult { - guidance::setting_help(py, options) + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "setting_help", + options, + (&["purpose", "file_name", "section", "key", "value"], &[]), + )?; + let text = |name: &str| -> PyResult { arguments.value(name)?.extract() }; + let (file_name, section, key): (String, String, String) = + (text("file_name")?, text("section")?, text("key")?); + Ok(setting_help_text( + &text("purpose")?, + SettingSite { + file_name: &file_name, + section: §ion, + key: &key, + }, + &setting_text(&arguments.value("value")?)?, + )) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/failures/sql_analysis_guidance.rs b/crates/sqlbuild-python-front/src/discovery/main/failures/sql_analysis_guidance.rs index 4073b59569..003f8d47b9 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/failures/sql_analysis_guidance.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/failures/sql_analysis_guidance.rs @@ -1,15 +1,25 @@ //! The note and help for turning SQL analysis on, or `None` when it is effectively on. -use crate::discovery::_helpers::failures::guidance; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::guidance::SQL_ANALYSIS_KEY; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_config::project::main::sql_analysis_off_guidance::sql_analysis_off_guidance as analysis_off; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The note and help for turning SQL analysis on, or `None` when it is effectively on. -#[inline] pub fn sql_analysis_guidance( discovered_inputs: &Bound<'_, PyAny>, purpose: &str, ) -> PyResult> { - guidance::sql_analysis_guidance(discovered_inputs, purpose) + let local: Bound<'_, PyAny> = native_field(discovered_inputs, "local_config")?; + let local_override: bool = + native_field(&local, "setting_overrides")?.contains(SQL_ANALYSIS_KEY)?; + let owner: Bound<'_, PyAny> = if local_override { + local + } else { + native_field(discovered_inputs, "project_config")? + }; + if native_field(&native_field(&owner, "settings")?, SQL_ANALYSIS_KEY)?.is_truthy()? { + return Ok(None); + } + Ok(Some(analysis_off(local_override, purpose))) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/hook_entries.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/hook_entries.rs index 5467e5d106..6c12c31e92 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/headers/hook_entries.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/hook_entries.rs @@ -1,18 +1,42 @@ //! Serialize SQL and Python hook entries; other values serialize to no hooks. -use crate::discovery::_helpers::headers::hook_entries; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyList; +use crate::discovery::_helpers::headers::hook_entries::sql_hook; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// Serialize SQL and Python hook entries; other values serialize to no hooks. -#[inline] pub fn hook_entries<'py>( value: &Bound<'py, PyAny>, sql_fields: &[String], python_hook_fields: Option<&Bound<'py, PyDict>>, ) -> PyResult> { - hook_entries::hook_entries(value, sql_fields, python_hook_fields) + let py = value.py(); + let hooks: Bound<'py, PyList> = PyList::empty(py); + if !value.is_instance_of::() && !value.is_instance_of::() { + return Ok(hooks); + } + let sql_class: Bound<'py, PyAny> = record_class(py, RecordId::SqlHookEntry)?; + let python_class: Bound<'py, PyAny> = record_class(py, RecordId::PythonHookEntry)?; + for entry in value.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + if entry.is_instance(&sql_class)? { + hooks.append(sql_hook(&entry, sql_fields)?)?; + } else if entry.is_instance(&python_class)? { + let hook: Bound<'py, PyDict> = PyDict::new(py); + let name: Bound<'py, PyAny> = native_field(&entry, "name")?; + hook.set_item("type", "python")?; + hook.set_item("name", &name)?; + hook.set_item("kwargs", native_field(&entry, "kwargs")?)?; + if let Some(fields) = python_hook_fields + && let Some(extra) = fields.get_item(&name)? + { + hook.call_method1("update", (extra,))?; + } + hooks.append(hook)?; + } + } + Ok(hooks) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_declarations.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_declarations.rs index fda14402b4..086a5a861b 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_declarations.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_declarations.rs @@ -1,12 +1,10 @@ //! Register the header declarations functions and classes. -use crate::discovery::_helpers::headers::header_declarations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::headers::header_declarations::model_header_declarations; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the header declarations functions and classes. -#[inline] pub fn register_header_declarations(module: &Bound<'_, PyModule>) -> PyResult<()> { - header_declarations::register(module) + module.add_function(wrap_pyfunction!(model_header_declarations, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs index ae668cb5c5..48cc7c0303 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs @@ -1,12 +1,14 @@ //! Register the header values functions and classes. -use crate::discovery::_helpers::headers::header_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::headers::header_values::model_output_column_locations; +use crate::discovery::_helpers::headers::header_values::parse_header_values; +use crate::discovery::_helpers::headers::header_values::prepare_model_header_tokens; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the header values functions and classes. -#[inline] pub fn register_header_values(module: &Bound<'_, PyModule>) -> PyResult<()> { - header_values::register(module) + module.add_function(wrap_pyfunction!(prepare_model_header_tokens, module)?)?; + module.add_function(wrap_pyfunction!(parse_header_values, module)?)?; + module.add_function(wrap_pyfunction!(model_output_column_locations, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/register_hook_entries.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/register_hook_entries.rs index cd409be58a..b763d2f151 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/headers/register_hook_entries.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/register_hook_entries.rs @@ -1,12 +1,11 @@ //! Register the hook entries functions and classes. -use crate::discovery::_helpers::headers::hook_entries; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::headers::hook_entries::serialize_hook_entries; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the hook entries functions and classes. -#[inline] pub fn register_hook_entries(module: &Bound<'_, PyModule>) -> PyResult<()> { - hook_entries::register(module) + module.add_function(wrap_pyfunction!(serialize_hook_entries, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/headers/register_key_checks.rs b/crates/sqlbuild-python-front/src/discovery/main/headers/register_key_checks.rs index c09f4fc734..bc5dea63d9 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/headers/register_key_checks.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/headers/register_key_checks.rs @@ -1,12 +1,12 @@ //! Register the key checks functions and classes. -use crate::discovery::_helpers::headers::key_checks; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::headers::key_checks::unknown_mapping_keys; +use crate::discovery::_helpers::headers::key_checks::unsupported_keys_help; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the key checks functions and classes. -#[inline] pub fn register_key_checks(module: &Bound<'_, PyModule>) -> PyResult<()> { - key_checks::register(module) + module.add_function(wrap_pyfunction!(unknown_mapping_keys, module)?)?; + module.add_function(wrap_pyfunction!(unsupported_keys_help, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/column_name.rs b/crates/sqlbuild-python-front/src/discovery/main/project/column_name.rs index 6c6a3da67e..9a678b15df 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/column_name.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/column_name.rs @@ -1,12 +1,9 @@ //! Column name. -use crate::discovery::_helpers::project::project_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Column name. -#[inline] pub fn column_name(column: &Bound<'_, PyAny>) -> PyResult { - project_facts::column_name(column) + native_field(column, "name")?.extract() } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/configured_changes.rs b/crates/sqlbuild-python-front/src/discovery/main/project/configured_changes.rs index c647110437..a0628da829 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/configured_changes.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/configured_changes.rs @@ -1,13 +1,11 @@ //! `[changes]` of the project at `project_dir`: what `--changed` counts and reaches by default. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::project::project_config::configured; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_config::project::main::load_changes_config::load_changes_config; use sqlbuild_config::project::models::ChangesConfig; /// `[changes]` of the project at `project_dir`: what `--changed` counts and reaches by default. -#[inline] pub fn configured_changes(project_dir: &Bound<'_, PyAny>) -> PyResult { - project_config::configured_changes(project_dir) + configured(project_dir, load_changes_config) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/configured_git_base_ref.rs b/crates/sqlbuild-python-front/src/discovery/main/project/configured_git_base_ref.rs index 453f1ecffe..cc3ee70351 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/configured_git_base_ref.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/configured_git_base_ref.rs @@ -1,12 +1,10 @@ //! `[git] base_ref` of the project at `project_dir`, which `--since @base` names. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::project::project_config::configured; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_config::project::main::load_git_base_ref::load_git_base_ref; /// `[git] base_ref` of the project at `project_dir`, which `--since @base` names. -#[inline] pub fn configured_git_base_ref(project_dir: &Bound<'_, PyAny>) -> PyResult> { - project_config::configured_git_base_ref(project_dir) + configured(project_dir, load_git_base_ref) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/configured_review.rs b/crates/sqlbuild-python-front/src/discovery/main/project/configured_review.rs index 37b4f3abc4..e683ec6c8a 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/configured_review.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/configured_review.rs @@ -1,13 +1,11 @@ //! `[review]` of the project at `project_dir`: the defaults `sqb review` falls back to. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::project::project_config::configured; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_config::project::main::load_review_config::load_review_config; use sqlbuild_config::project::models::ReviewConfig; /// `[review]` of the project at `project_dir`: the defaults `sqb review` falls back to. -#[inline] pub fn configured_review(project_dir: &Bound<'_, PyAny>) -> PyResult { - project_config::configured_review(project_dir) + configured(project_dir, load_review_config) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/discover_configured_providers.rs b/crates/sqlbuild-python-front/src/discovery/main/project/discover_configured_providers.rs index 5a7dc7607d..6e4457a04a 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/discover_configured_providers.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/discover_configured_providers.rs @@ -1,18 +1,41 @@ //! The configuration with discovery's checks and warnings, and the provider classes, without importing any other project Python. -use crate::discovery::_helpers::project::configuration; +use crate::compile_driver::main::reject_invalid_project_configuration::reject_invalid_project_configuration; +use crate::discovery::_helpers::project::configuration::loaded_configuration; +use crate::discovery::_helpers::python_files::imported_providers::discover_provider_classes; +use crate::discovery::main::project::warn_deprecated_settings::warn_deprecated_settings; use crate::discovery::models::SecretPolicy; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// The configuration with discovery's checks and warnings, and the provider classes, without importing any other project Python. -#[inline] pub fn discover_configured_providers<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, secret_policy: SecretPolicy, ) -> PyResult> { - configuration::discover_configured_providers(py, project_dir, secret_policy) + let configuration: Bound<'py, PyAny> = loaded_configuration(py, project_dir, |config| { + reject_invalid_project_configuration(config)?; + warn_deprecated_settings(py, project_dir) + })?; + let providers: Bound<'py, PyAny> = + discover_provider_classes(project_dir.clone(), secret_policy.name())?; + new_record_with( + py, + RecordId::DiscoveredProjectInputs, + &[ + ("project_dir", project_dir.clone()), + ( + "project_config", + native_field(&configuration, "project_config")?, + ), + ( + "local_config", + native_field(&configuration, "local_config")?, + ), + ("providers", providers), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/discover_connection_configuration.rs b/crates/sqlbuild-python-front/src/discovery/main/project/discover_connection_configuration.rs index 72edfe0943..caae7dab5d 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/discover_connection_configuration.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/discover_connection_configuration.rs @@ -1,16 +1,13 @@ //! The validated configuration without a discovery lifecycle, which reading two files never needs. -use crate::discovery::_helpers::project::configuration; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::compile_driver::main::reject_invalid_connection_configuration::reject_invalid_connection_configuration; +use crate::discovery::_helpers::project::configuration::loaded_configuration; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; /// The validated configuration without a discovery lifecycle, which reading two files never needs. -#[inline] pub fn discover_connection_configuration<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, ) -> PyResult> { - configuration::discover_connection_configuration(py, project_dir) + loaded_configuration(py, project_dir, reject_invalid_connection_configuration) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/discover_project_configuration.rs b/crates/sqlbuild-python-front/src/discovery/main/project/discover_project_configuration.rs index cec76516c8..1e1144fe5a 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/discover_project_configuration.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/discover_project_configuration.rs @@ -1,16 +1,12 @@ //! `DiscoveredProjectInputs` with the loaded configuration, inside the project discovery lifecycle. -use crate::discovery::_helpers::project::configuration; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::discovery::_helpers::project::configuration::configuration_inputs; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; /// `DiscoveredProjectInputs` with the loaded configuration, inside the project discovery lifecycle. -#[inline] pub fn discover_project_configuration<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, ) -> PyResult> { - configuration::discover_project_configuration(py, project_dir) + configuration_inputs(py, project_dir, |_| Ok(())) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/each.rs b/crates/sqlbuild-python-front/src/discovery/main/project/each.rs index 55055191f5..3eae8b954e 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/each.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/each.rs @@ -1,9 +1,7 @@ //! `read` applied to each item of the iterable attribute `name` of `owner`. use crate::discovery::_helpers::project::project_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// `read` applied to each item of the iterable attribute `name` of `owner`. #[inline] diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/load_local_config_file.rs b/crates/sqlbuild-python-front/src/discovery/main/project/load_local_config_file.rs index cf145a6ad4..689f5f745d 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/load_local_config_file.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/load_local_config_file.rs @@ -1,17 +1,21 @@ //! `load_local_config`: the validated `LocalConfig` of `project_dir`, or the default. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::discovery::_helpers::project::project_config::built; +use crate::discovery::_helpers::project::project_config::files; +use crate::discovery::_helpers::project::project_config::load_error; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_config::project::main::load_local_config::load_local_config; /// `load_local_config`: the validated `LocalConfig` of `project_dir`, or the default. -#[inline] +#[pyfunction] pub fn load_local_config_file( py: Python<'_>, project_dir: &Bound<'_, PyAny>, ) -> PyResult> { - project_config::load_local_config_file(py, project_dir) + let files = files(project_dir)?; + match load_local_config(&files.native) { + Ok(config) => built(py, config), + Err(failure) => Err(load_error(py, &failure, &files)?), + } } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/load_project_config_file.rs b/crates/sqlbuild-python-front/src/discovery/main/project/load_project_config_file.rs index e72087f8ec..8d3ce64549 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/load_project_config_file.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/load_project_config_file.rs @@ -1,17 +1,21 @@ //! `load_project_config`: the validated `ProjectConfig` of `project_dir`. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::discovery::_helpers::project::project_config::built; +use crate::discovery::_helpers::project::project_config::files; +use crate::discovery::_helpers::project::project_config::load_error; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_config::project::main::load_project_config::load_project_config; /// `load_project_config`: the validated `ProjectConfig` of `project_dir`. -#[inline] +#[pyfunction] pub fn load_project_config_file( py: Python<'_>, project_dir: &Bound<'_, PyAny>, ) -> PyResult> { - project_config::load_project_config_file(py, project_dir) + let files = files(project_dir)?; + match load_project_config(&files.native) { + Ok(config) => built(py, config), + Err(failure) => Err(load_error(py, &failure, &files)?), + } } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/named_node_rows.rs b/crates/sqlbuild-python-front/src/discovery/main/project/named_node_rows.rs index c7291aa22e..37581a4ac6 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/named_node_rows.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/named_node_rows.rs @@ -1,13 +1,25 @@ //! The discovered Python nodes by kind, name and path only, as the resource-name check reads them. -use crate::discovery::_helpers::project::project_facts; +use crate::discovery::_helpers::project::project_facts::PYTHON_NODE_FIELDS; +use crate::discovery::_helpers::project::project_facts::each; +use crate::discovery::_helpers::project::project_facts::text; use crate::discovery::models::NodeRow; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The discovered Python nodes by kind, name and path only, as the resource-name check reads them. -#[inline] pub fn named_node_rows(inputs: &Bound<'_, PyAny>) -> PyResult> { - project_facts::named_node_rows(inputs) + let mut rows: Vec = Vec::new(); + for (kind, name) in PYTHON_NODE_FIELDS { + rows.extend(each(inputs, name, |node| { + Ok(NodeRow { + kind: kind.to_owned(), + name: native_field(node, "name")?.extract()?, + path: text(node, "relative_path")?, + decorator_config: false, + depends_on: Vec::new(), + }) + })?); + } + Ok(rows) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/project_facts.rs b/crates/sqlbuild-python-front/src/discovery/main/project/project_facts.rs index 4cd226e6e7..a3bd94c780 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/project_facts.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/project_facts.rs @@ -1,19 +1,63 @@ //! The facts of `inputs`, with the Python nodes the facade has described. -use crate::discovery::_helpers::project::project_facts; +use crate::discovery::_helpers::project::project_facts::audit_fact; +use crate::discovery::_helpers::project::project_facts::csv_field_limit; +use crate::discovery::_helpers::project::project_facts::declarations_fact; +use crate::discovery::_helpers::project::project_facts::each; +use crate::discovery::_helpers::project::project_facts::model_fact; +use crate::discovery::_helpers::project::project_facts::named; +use crate::discovery::_helpers::project::project_facts::node_fact; +use crate::discovery::_helpers::project::project_facts::schema_fact; +use crate::discovery::_helpers::project::project_facts::seed_file_fact; +use crate::discovery::_helpers::project::project_facts::source_file_fact; +use crate::discovery::_helpers::project::project_facts::stem_named; +use crate::discovery::_helpers::project::project_facts::test_file_fact; use crate::discovery::models::NodeRow; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use sqlbuild_discovery::project_validation::models::ProjectFacts; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_discovery::project_validation::models::{ + NamedPath, ProjectFacts, ScopedDeclarationsFact, +}; +use sqlbuild_python_base::compile_context::main::delta::injection::injection; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The facts of `inputs`, with the Python nodes the facade has described. -#[inline] pub fn project_facts( py: Python<'_>, inputs: &Bound<'_, PyAny>, nodes: Vec, ) -> PyResult { - project_facts::project_facts(py, inputs, nodes) + let mut declaration_files: Vec = + each(inputs, "enum_files", declarations_fact)?; + declaration_files.extend(each(inputs, "constant_files", declarations_fact)?); + let mut model_schemas: Vec = Vec::new(); + for schema in native_field(inputs, "model_schema_files")?.try_iter()? { + model_schemas.extend(each(&schema?, "declarations", named)?); + } + Ok(ProjectFacts { + models: each(inputs, "model_files", model_fact)?, + sql_functions: each(inputs, "sql_function_files", stem_named)?, + python_functions: each(inputs, "python_function_files", stem_named)?, + sql_hooks: each(inputs, "sql_hook_files", named)?, + seeds: each(inputs, "seed_files", seed_file_fact)?, + schemas: each(inputs, "schema_files", |schema| schema_fact(py, schema))?, + sources: each(inputs, "source_files", source_file_fact)?, + tests: each(inputs, "test_files", test_file_fact)?, + scenarios: each(inputs, "scenario_files", named)?, + audits: each(inputs, "audit_files", audit_fact)?, + declaration_files, + model_schemas, + materializations: each(inputs, "materialization_files", named)?, + nodes: nodes.into_iter().map(node_fact).collect::>()?, + hook_functions: each(inputs, "hook_functions", named)?, + event_exporters: each(inputs, "event_exporters", named)?, + providers: each(inputs, "providers", named)?, + path_default_keys: native_field(inputs, "project_config")? + .getattr("path_defaults")? + .try_iter()? + .map(|key| key?.extract()) + .collect::>()?, + project_model_paths: injection().map(|injection| injection.project_model_paths.clone()), + csv_field_limit: Some(csv_field_limit(py)?), + orchestration: Vec::new(), + }) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/register_locations.rs b/crates/sqlbuild-python-front/src/discovery/main/project/register_locations.rs index 8765670175..0bc99fcf13 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/register_locations.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/register_locations.rs @@ -1,12 +1,13 @@ //! Register the locations functions and classes. -use crate::discovery::_helpers::project::locations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::project::locations::project_relative_path; +use crate::discovery::main::project::yaml_entry_line::yaml_entry_line; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the locations functions and classes. -#[inline] pub fn register_locations(module: &Bound<'_, PyModule>) -> PyResult<()> { - locations::register(module) + module.add_function(wrap_pyfunction!(yaml_entry_line, module)?)?; + module.add_function(wrap_pyfunction!(project_relative_path, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs index 065654f6f6..1717f22fe1 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs @@ -1,12 +1,16 @@ //! Register the project config functions and classes. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::main::project::load_local_config_file::load_local_config_file; +use crate::discovery::main::project::load_project_config_file::load_project_config_file; +use crate::discovery::main::project::resolve_config_file_path::resolve_config_file_path; +use crate::discovery::main::project::validate_local_sql_analysis::validate_local_sql_analysis; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the project config functions and classes. -#[inline] pub fn register_project_config(module: &Bound<'_, PyModule>) -> PyResult<()> { - project_config::register(module) + module.add_function(wrap_pyfunction!(load_project_config_file, module)?)?; + module.add_function(wrap_pyfunction!(load_local_config_file, module)?)?; + module.add_function(wrap_pyfunction!(resolve_config_file_path, module)?)?; + module.add_function(wrap_pyfunction!(validate_local_sql_analysis, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs index 117895dbe5..009817cbf6 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs @@ -1,12 +1,15 @@ //! Whether native discovery reproduces the string semantics of this Python and Unicode version. -use crate::discovery::_helpers::project::project_files; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::project::project_files::parse_model_contents; +use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; +use crate::discovery::models::NativeModelFile; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Whether native discovery reproduces the string semantics of this Python and Unicode version. -#[inline] pub fn register_project_files(module: &Bound<'_, PyModule>) -> PyResult<()> { - project_files::register(module) + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(parse_model_contents, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/register_project_validation.rs b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_validation.rs index b2f59cb128..b29729cdea 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/register_project_validation.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/register_project_validation.rs @@ -1,12 +1,13 @@ //! Register the project validation functions and classes. -use crate::discovery::_helpers::project::project_validation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::project::project_validation::validate_discovered_project_inputs; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the project validation functions and classes. -#[inline] pub fn register_project_validation(module: &Bound<'_, PyModule>) -> PyResult<()> { - project_validation::register(module) + module.add_function(wrap_pyfunction!( + validate_discovered_project_inputs, + module + )?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/resolve_config_file_path.rs b/crates/sqlbuild-python-front/src/discovery/main/project/resolve_config_file_path.rs index 72d5f0368a..b241d7adc3 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/resolve_config_file_path.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/resolve_config_file_path.rs @@ -1,17 +1,22 @@ //! The project or local configuration file path, rejecting an unconverted legacy file. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::discovery::_helpers::project::project_config::config_error; +use crate::discovery::_helpers::project::project_config::file_index; +use crate::discovery::_helpers::project::project_config::files; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_config::project::main::resolve_config_file::resolve_config_file; /// The project or local configuration file path, rejecting an unconverted legacy file. -#[inline] +#[pyfunction] pub fn resolve_config_file_path<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, local: bool, ) -> PyResult> { - project_config::resolve_config_file_path(py, project_dir, local) + let files = files(project_dir)?; + match resolve_config_file(&files.native, local) { + Ok(file) => Ok(files.python[file_index(file)].clone()), + Err(failure) => Err(config_error(py, &failure, &files)?), + } } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/seed_csv_settings.rs b/crates/sqlbuild-python-front/src/discovery/main/project/seed_csv_settings.rs index c8f03c42c9..c541f47b6b 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/seed_csv_settings.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/seed_csv_settings.rs @@ -1,13 +1,19 @@ //! The CSV reader settings a seed declaration authors, without the dialect check. -use crate::discovery::_helpers::project::project_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::project::project_facts::optional_text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_discovery::project_validation::models::CsvSettings; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The CSV reader settings a seed declaration authors, without the dialect check. -#[inline] pub fn seed_csv_settings(entry: &Bound<'_, PyAny>) -> PyResult { - project_facts::seed_csv_settings(entry) + let settings: Bound<'_, PyAny> = native_field(entry, "csv_settings")?; + Ok(CsvSettings { + delimiter: optional_text(&settings, "delimiter")?, + quotechar: optional_text(&settings, "quotechar")?, + escapechar: optional_text(&settings, "escapechar")?, + doublequote: native_field(&settings, "doublequote")?.extract()?, + skipinitialspace: native_field(&settings, "skipinitialspace")?.extract()?, + dialect_error: None, + }) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/stem.rs b/crates/sqlbuild-python-front/src/discovery/main/project/stem.rs index aadcc56de9..77509601f3 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/stem.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/stem.rs @@ -1,12 +1,10 @@ //! `str(Path(...).stem)` of the path attribute `name`. -use crate::discovery::_helpers::project::project_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::path_stem::path_stem; /// `str(Path(...).stem)` of the path attribute `name`. -#[inline] pub fn stem(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { - project_facts::stem(owner, name) + path_stem(&native_field(owner, name)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/validate_local_sql_analysis.rs b/crates/sqlbuild-python-front/src/discovery/main/project/validate_local_sql_analysis.rs index 9b0eb51bbd..b002613ca3 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/validate_local_sql_analysis.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/validate_local_sql_analysis.rs @@ -1,17 +1,21 @@ //! `validate_local_sql_analysis_policy` from the project requirement and the local setting. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyErr; -use pyo3::prelude::Python; +use crate::discovery::_helpers::project::project_config::config_error; +use crate::discovery::_helpers::project::project_config::files; +use pyo3::prelude::{Bound, PyAny, PyErr, Python}; +use pyo3::pyfunction; +use sqlbuild_config::project::main::local_sql_analysis_failure::local_sql_analysis_failure; /// `validate_local_sql_analysis_policy` from the project requirement and the local setting. -#[inline] +#[pyfunction] pub fn validate_local_sql_analysis( py: Python<'_>, project_dir: &Bound<'_, PyAny>, flags: (bool, bool, bool), ) -> Result<(), PyErr> { - project_config::validate_local_sql_analysis(py, project_dir, flags) + let files = files(project_dir)?; + match local_sql_analysis_failure(&files.native, flags) { + None => Ok(()), + Some(failure) => Err(config_error(py, &failure, &files)?), + } } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/validation_error.rs b/crates/sqlbuild-python-front/src/discovery/main/project/validation_error.rs index f8785d84ce..8e420b2c35 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/validation_error.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/validation_error.rs @@ -1,12 +1,45 @@ //! The Python exception a failed project validation raises. -use crate::discovery::_helpers::project::project_validation; -use pyo3::prelude::PyErr; -use pyo3::prelude::Python; +use crate::discovery::_helpers::failures::raised::{failure_error, read_error}; +use crate::discovery::_helpers::project::project_validation::BUILTINS_MODULE; +use crate::discovery::_helpers::project::project_validation::CSV_DIALECT_ERRORS; +use crate::discovery::_helpers::project::project_validation::CSV_MODULE; +use crate::discovery::_helpers::project::project_validation::PATHLIB_MODULE; +use crate::discovery::_helpers::project::project_validation::raised; +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{PyAnyMethods, PyErr, Python}; use sqlbuild_discovery::project_validation::models::ValidationFailure; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The Python exception a failed project validation raises. -#[inline] pub fn validation_error(py: Python<'_>, failure: ValidationFailure) -> PyErr { - project_validation::validation_error(py, failure) + match failure { + ValidationFailure::Discovery(failure) => failure_error(py, failure), + ValidationFailure::Unreadable { + absolute_path, + failure, + } => match py + .import(PATHLIB_MODULE) + .and_then(|module| native_field(&module, "Path")) + .and_then(|path| path.call1((absolute_path,))) + { + Ok(file_path) => read_error(failure, &file_path), + Err(error) => error, + }, + ValidationFailure::Csv(message) => raised( + py.import(CSV_MODULE) + .and_then(|module| native_field(&module, "Error")) + .and_then(|class| class.call1((message,))), + ), + ValidationFailure::CsvDialect(error) => { + if !CSV_DIALECT_ERRORS.contains(&error.class.as_str()) { + return PyKeyError::new_err(error.class.to_string()); + } + raised( + py.import(BUILTINS_MODULE) + .and_then(|module| native_field(&module, error.class.as_str())) + .and_then(|class| class.call1((error.message,))), + ) + } + } } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/warn_deprecated_settings.rs b/crates/sqlbuild-python-front/src/discovery/main/project/warn_deprecated_settings.rs index d3b3a3bf12..b97eb0c921 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/warn_deprecated_settings.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/warn_deprecated_settings.rs @@ -1,13 +1,18 @@ //! Write a warning line to stderr for each deprecated setting alias `project_dir` still sets. -use crate::discovery::_helpers::project::project_config; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::discovery::_helpers::project::project_config::files; +use crate::discovery::_helpers::project::project_config::load_error; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_config::project::main::deprecated_setting_warnings::deprecated_setting_warnings; +use sqlbuild_python_base::native_host::main::write_console::write_console; /// Write a warning line to stderr for each deprecated setting alias `project_dir` still sets. -#[inline] pub fn warn_deprecated_settings(py: Python<'_>, project_dir: &Bound<'_, PyAny>) -> PyResult<()> { - project_config::warn_deprecated_settings(py, project_dir) + let files = files(project_dir)?; + match deprecated_setting_warnings(&files.native) { + Ok(warnings) => warnings + .iter() + .try_for_each(|warning| write_console(py, "stderr", &format!("{warning}\n"))), + Err(failure) => Err(load_error(py, &failure, &files)?), + } } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/yaml_entry_line.rs b/crates/sqlbuild-python-front/src/discovery/main/project/yaml_entry_line.rs index b524063a91..65d6a7bf7c 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/yaml_entry_line.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/yaml_entry_line.rs @@ -1,9 +1,13 @@ //! The 1-based line of the outermost `- name: ` entry, if present. -use crate::discovery::_helpers::project::locations; +use pyo3::pyfunction; +use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; /// The 1-based line of the outermost `- name: ` entry, if present. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, contents, name))] pub fn yaml_entry_line(contents: &str, name: &str) -> Option { - locations::yaml_entry_line(contents, name) + yaml_entry_lines(contents) + .into_iter() + .find_map(|(entry, line)| (entry == name).then_some(line)) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_event_exporter_functions.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_event_exporter_functions.rs index acb768a560..bf94e79267 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_event_exporter_functions.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_event_exporter_functions.rs @@ -1,15 +1,36 @@ //! Validated lifecycle-event sink declarations under sinks/, bound to providers; Recall replacement sinks are left out. -use crate::discovery::_helpers::python_files::imported_providers; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::python_files::imported_modules::{ProjectDir, tuple_of}; +use crate::discovery::_helpers::python_files::imported_providers::bind_event_exporter_declarations; +use crate::discovery::_helpers::python_files::imported_providers::discover_event_exporter_declarations; +use crate::discovery::_helpers::python_files::imported_providers::takes_recall_writer; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; /// Validated lifecycle-event sink declarations under sinks/, bound to providers; Recall replacement sinks are left out. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, providers = None))] pub fn discover_event_exporter_functions<'py>( project_dir: Bound<'py, PyAny>, providers: Option>, ) -> PyResult> { - imported_providers::discover_event_exporter_functions(project_dir, providers) + let py: Python<'py> = project_dir.py(); + let project = ProjectDir { + path: project_dir.clone(), + }; + if let Some(cached) = project.cached_extensions()? { + return cached.get_item(1); + } + let mut ordinary: Vec> = Vec::new(); + for declaration in discover_event_exporter_declarations(project_dir.clone())?.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + if !takes_recall_writer(&declaration)? { + ordinary.push(declaration); + } + } + bind_event_exporter_declarations( + tuple_of(py, ordinary)?, + providers.map_or_else(|| tuple_of(py, Vec::new()), Ok)?, + project_dir, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_hook_functions.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_hook_functions.rs index f046e4db10..ff31f4be65 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_hook_functions.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_hook_functions.rs @@ -1,9 +1,7 @@ //! Decorated model lifecycle hook functions under hooks/python/. use crate::discovery::_helpers::python_files::imported_python_nodes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Decorated model lifecycle hook functions under hooks/python/. #[inline] diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_materialization_files.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_materialization_files.rs index 2b665bfcc1..e370b7e2c0 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_materialization_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_materialization_files.rs @@ -1,15 +1,56 @@ //! Custom materialization Python files under materializations/. -use crate::discovery::_helpers::python_files::imported_python_nodes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::python_files::imported_modules::{ + ProjectDir, discovered, tuple_of, +}; +use crate::discovery::main::python_files::provider_by_name::provider_by_name; +use crate::discovery::main::python_files::provider_usages::provider_usages; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// Custom materialization Python files under materializations/. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, providers = None))] pub fn discover_materialization_files<'py>( project_dir: Bound<'py, PyAny>, providers: Option>, ) -> PyResult> { - imported_python_nodes::discover_materialization_files(project_dir, providers) + let py: Python<'py> = project_dir.py(); + let project = ProjectDir { path: project_dir }; + if !project + .child("materializations")? + .call_method0("is_dir")? + .is_truthy()? + { + return tuple_of(py, Vec::new()); + } + let providers: Bound<'py, PyDict> = match providers { + Some(providers) => provider_by_name(&providers)?, + None => PyDict::new(py), + }; + let mut files: Vec> = Vec::new(); + for file_path in project.module_files("materializations", false)? { + let materialize: Bound<'py, PyAny> = + project.load(UserCodeHost::LoadMaterializeFunction, &file_path)?; + let usages: Bound<'py, PyAny> = if materialize.is_none() { + tuple_of(py, Vec::new())? + } else { + provider_usages(&materialize, &providers)? + }; + files.push(discovered( + py, + RecordId::DiscoveredMaterializationFile, + &[ + ("file_path", file_path.clone()), + ("relative_path", project.relative(&file_path)?), + ("name", native_field(&file_path, "stem")?), + ("provider_usages", usages), + ], + )?); + } + tuple_of(py, files) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_provider_classes.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_provider_classes.rs index c261a096ca..7c4625cbfc 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_provider_classes.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_provider_classes.rs @@ -1,9 +1,7 @@ //! Provider classes under providers/. use crate::discovery::_helpers::python_files::imported_providers; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Provider classes under providers/. #[inline] diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_function_files.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_function_files.rs index 89093f5de4..2cb0f9887c 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_function_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_function_files.rs @@ -1,15 +1,34 @@ //! Discover the `@udf(...)` Python function files under `functions/python/`. -use crate::discovery::_helpers::python_files::python_function_files; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::python_files::python_function_files::function_record; +use crate::discovery::_helpers::snapshots::native_classes::NativeProjectTree; +use crate::discovery::_helpers::snapshots::snapshot::{in_snapshot, project_root, project_tree}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::main::python_function_files::python_function_files; +use sqlbuild_discovery::models::{DiscoveredFile, ProjectRoot}; +use sqlbuild_discovery::tree::models::ProjectTree; /// Discover the `@udf(...)` Python function files under `functions/python/`. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir))] pub fn discover_python_function_files<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { - python_function_files::discover_python_function_files(project_dir) + in_snapshot(project_dir, || { + let py: Python<'py> = project_dir.py(); + let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; + let root: ProjectRoot = project_root(project_dir)?; + let inner: &ProjectTree = &tree.get().inner; + let files: Vec> = py + .detach(|| python_function_files(&root, inner)) + .map_err(|failure| stage_error(inner, project_dir, failure))?; + let records: Vec> = files + .into_iter() + .map(|file| function_record(project_dir, inner, file)) + .collect::>()?; + PyTuple::new(py, records) + }) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs index ab6f9875ce..446f0a8e07 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs @@ -1,15 +1,73 @@ //! Decorated Python DAG node functions under python/. -use crate::discovery::_helpers::python_files::imported_python_nodes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::python_files::imported_modules::{ + ProjectDir, discovered, shim, tuple_of, +}; +use crate::discovery::_helpers::python_files::imported_python_nodes::Buckets; +use crate::discovery::main::python_files::provider_by_name::provider_by_name; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_core::constants::PYTHON_NODE_ROOT; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::main::within_identity_pass::within_identity_pass; +use sqlbuild_python_base::records::types::RecordId; /// Decorated Python DAG node functions under python/. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, providers = None))] pub fn discover_python_node_functions<'py>( project_dir: Bound<'py, PyAny>, providers: Option>, ) -> PyResult> { - imported_python_nodes::discover_python_node_functions(project_dir, providers) + let py: Python<'py> = project_dir.py(); + let project = ProjectDir { path: project_dir }; + let providers: Bound<'py, PyDict> = match providers { + Some(providers) => provider_by_name(&providers)?, + None => PyDict::new(py), + }; + let mut buckets = Buckets { + nodes: [Vec::new(), Vec::new(), Vec::new(), Vec::new()], + audit_factories: Vec::new(), + }; + let node_root: Bound<'py, PyAny> = project.child(PYTHON_NODE_ROOT)?; + if node_root.call_method0("is_dir")?.is_truthy()? { + let saved: Bound<'py, PyAny> = call_with( + &shim(py, UserCodeHost::InstallPythonRootPackage)?, + &[ + ("node_root", node_root), + ("project_dir", project.path.clone()), + ], + )?; + let discovered_nodes: PyResult<()> = + within_identity_pass(py, || buckets.discover_nodes(&project, &providers)); + call_with( + &shim(py, UserCodeHost::RestorePythonRootModules)?, + &[("saved_modules", saved)], + )?; + discovered_nodes?; + } + let [loaders, tasks, assets, checks] = buckets.nodes; + discovered( + py, + RecordId::DiscoveredPythonNodeFunctions, + &[ + ("loaders", tuple_of(py, loaders)?), + ("tasks", tuple_of(py, tasks)?), + ("assets", tuple_of(py, assets)?), + ("checks", tuple_of(py, checks)?), + ( + "audit_factories", + tuple_of( + py, + buckets + .audit_factories + .into_iter() + .map(|(_, _, record)| record) + .collect(), + )?, + ), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/install_recall_writer.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/install_recall_writer.rs index 7cfe250281..afc6216ac9 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/install_recall_writer.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/install_recall_writer.rs @@ -1,11 +1,10 @@ //! Install the `RecallWriter` class sink parameters are recognised by, once, at registration. -use crate::discovery::_helpers::python_files::imported_providers; +use crate::discovery::_helpers::python_files::imported_providers::RECALL_WRITER; use pyo3::prelude::Bound; use pyo3::types::PyType; /// Install the `RecallWriter` class sink parameters are recognised by, once, at registration. -#[inline] pub fn install_recall_writer(writer: &Bound<'_, PyType>) { - imported_providers::install_recall_writer(writer) + let _ = RECALL_WRITER.get_or_init(writer.py(), || writer.clone().unbind()); } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/node_identity.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/node_identity.rs index 49f66190e2..5a49c753d8 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/node_identity.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/node_identity.rs @@ -1,10 +1,7 @@ //! The identity of one discovered node of `(kind, documented)`, hashed once by the user-code host. use crate::discovery::_helpers::python_files::imported_python_nodes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; /// The identity of one discovered node of `(kind, documented)`, hashed once by the user-code host. #[inline] diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs index 24098feb21..b38ff8467d 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs @@ -1,13 +1,15 @@ //! `{provider.name: provider for provider in providers}`. -use crate::discovery::_helpers::python_files::imported_modules; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// `{provider.name: provider for provider in providers}`. -#[inline] pub fn provider_by_name<'py>(providers: &Bound<'py, PyAny>) -> PyResult> { - imported_modules::provider_by_name(providers) + let by_name: Bound<'py, PyDict> = PyDict::new(providers.py()); + for provider in providers.try_iter()? { + let provider: Bound<'py, PyAny> = provider?; + by_name.set_item(native_field(&provider, "name")?, provider)?; + } + Ok(by_name) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_usages.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_usages.rs index 5685f5b30d..2b39ba1923 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_usages.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/provider_usages.rs @@ -1,16 +1,21 @@ //! The provider parameters of one loaded user function. -use crate::discovery::_helpers::python_files::imported_modules; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::python_files::imported_modules::shim; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_python_base::boundary::main::call_with::call_with; /// The provider parameters of one loaded user function. -#[inline] pub fn provider_usages<'py>( function: &Bound<'py, PyAny>, provider_by_name: &Bound<'py, PyDict>, ) -> PyResult> { - imported_modules::provider_usages(function, provider_by_name) + call_with( + &shim(function.py(), UserCodeHost::ProviderUsages)?, + &[ + ("function", function.clone()), + ("provider_by_name", provider_by_name.clone().into_any()), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs index ea02864cab..7ba126ee98 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs @@ -1,12 +1,25 @@ //! Register the imported python nodes functions and classes. -use crate::discovery::_helpers::python_files::imported_python_nodes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::python_files::imported_providers::bind_event_exporter_declarations; +use crate::discovery::_helpers::python_files::imported_providers::discover_event_exporter_declarations; +use crate::discovery::_helpers::python_files::imported_providers::discover_provider_classes; +use crate::discovery::_helpers::python_files::imported_python_nodes::discover_hook_functions; +use crate::discovery::main::python_files::discover_event_exporter_functions::discover_event_exporter_functions; +use crate::discovery::main::python_files::discover_materialization_files::discover_materialization_files; +use crate::discovery::main::python_files::discover_python_node_functions::discover_python_node_functions; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the imported python nodes functions and classes. -#[inline] pub fn register_imported_python_nodes(module: &Bound<'_, PyModule>) -> PyResult<()> { - imported_python_nodes::register(module) + module.add_function(wrap_pyfunction!(discover_python_node_functions, module)?)?; + module.add_function(wrap_pyfunction!(discover_hook_functions, module)?)?; + module.add_function(wrap_pyfunction!(discover_materialization_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_provider_classes, module)?)?; + module.add_function(wrap_pyfunction!( + discover_event_exporter_declarations, + module + )?)?; + module.add_function(wrap_pyfunction!(bind_event_exporter_declarations, module)?)?; + module.add_function(wrap_pyfunction!(discover_event_exporter_functions, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_function_files.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_function_files.rs index 8913e46dae..ef78cd7906 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_function_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_function_files.rs @@ -1,12 +1,10 @@ //! Register the python function files functions and classes. -use crate::discovery::_helpers::python_files::python_function_files; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::main::python_files::discover_python_function_files::discover_python_function_files; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the python function files functions and classes. -#[inline] pub fn register_python_function_files(module: &Bound<'_, PyModule>) -> PyResult<()> { - python_function_files::register(module) + module.add_function(wrap_pyfunction!(discover_python_function_files, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_hook_files.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_hook_files.rs index 3c7f62435a..01dd4787f3 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_hook_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_hook_files.rs @@ -1,12 +1,10 @@ //! Register the python hook files functions and classes. -use crate::discovery::_helpers::python_files::python_hook_files; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::python_files::python_hook_files::discover_python_hook_files; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the python hook files functions and classes. -#[inline] pub fn register_python_hook_files(module: &Bound<'_, PyModule>) -> PyResult<()> { - python_hook_files::register(module) + module.add_function(wrap_pyfunction!(discover_python_hook_files, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_module_files.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_module_files.rs index 8939ee1aec..35f50d5eeb 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_module_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/register_python_module_files.rs @@ -1,12 +1,10 @@ //! Register the python module files functions and classes. -use crate::discovery::_helpers::python_files::python_module_files; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::python_files::python_module_files::discover_python_module_files; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the python module files functions and classes. -#[inline] pub fn register_python_module_files(module: &Bound<'_, PyModule>) -> PyResult<()> { - python_module_files::register(module) + module.add_function(wrap_pyfunction!(discover_python_module_files, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/python_files/reject_unsupported_python_paths.rs b/crates/sqlbuild-python-front/src/discovery/main/python_files/reject_unsupported_python_paths.rs index af1736586b..b176075c6d 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/python_files/reject_unsupported_python_paths.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/python_files/reject_unsupported_python_paths.rs @@ -1,12 +1,28 @@ //! Raise `ProjectPythonPathError` naming every unsupported Python file below `project_dir`. -use crate::discovery::_helpers::python_files::python_paths; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::raised::discovery_error; +use crate::discovery::_helpers::python_files::python_paths::PYTHON_PATH_HELP; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_discovery::python_paths::main::unsupported_python_paths::unsupported_python_paths; +use std::path::PathBuf; /// Raise `ProjectPythonPathError` naming every unsupported Python file below `project_dir`. -#[inline] pub fn reject_unsupported_python_paths(project_dir: &Bound<'_, PyAny>) -> PyResult<()> { - python_paths::reject_unsupported_python_paths(project_dir) + let root: PathBuf = project_dir.extract()?; + let unsupported: Vec = unsupported_python_paths(&root) + .into_iter() + .map(|parts| parts.join(std::path::MAIN_SEPARATOR_STR)) + .collect(); + if unsupported.is_empty() { + return Ok(()); + } + Err(discovery_error( + project_dir.py(), + "python_path", + format!( + "Unsupported project Python path(s): {}", + unsupported.join(", ") + ), + Some(PYTHON_PATH_HELP.to_owned()), + )) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_adapter_file.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_adapter_file.rs index e4962b97a9..4d5a007a95 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_adapter_file.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_adapter_file.rs @@ -1,12 +1,26 @@ //! Detect a project-level adapter.py without importing it. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::path_class; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; /// Detect a project-level adapter.py without importing it. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir))] pub fn discover_adapter_file<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { - declaration_records::discover_adapter_file(project_dir) + let py: Python<'py> = project_dir.py(); + let file_path: Bound<'py, PyAny> = project_dir.div("adapter.py")?; + if !file_path.call_method0("is_file")?.is_truthy()? { + return Ok(py.None().into_bound(py)); + } + new_record_with( + py, + RecordId::DiscoveredAdapterFile, + &[ + ("file_path", file_path), + ("relative_path", path_class(py)?.call1(("adapter.py",))?), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_audit_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_audit_files.rs index 19af629ec1..e68dd482e7 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_audit_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_audit_files.rs @@ -1,16 +1,22 @@ //! Discover generic and singular audit SQL files in their project-wide and scoped roles. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover generic and singular audit SQL files in their project-wide and scoped roles. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] pub fn discover_audit_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, ) -> PyResult> { - declaration_records::discover_audit_files(project_dir, on_fault) + discovered( + &faults(project_dir, on_fault), + CollectionKind::Audits, + false, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_constant_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_constant_files.rs index 05ec7b9656..828c180472 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_constant_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_constant_files.rs @@ -1,17 +1,23 @@ //! Discover global and scoped constant declaration files. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover global and scoped constant declaration files. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None, isolate_declaration_kind=false))] pub fn discover_constant_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, isolate_declaration_kind: bool, ) -> PyResult> { - declaration_records::discover_constant_files(project_dir, on_fault, isolate_declaration_kind) + discovered( + &faults(project_dir, on_fault), + CollectionKind::Constants, + isolate_declaration_kind, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_enum_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_enum_files.rs index d9eacb8305..cf228ee104 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_enum_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_enum_files.rs @@ -1,17 +1,23 @@ //! Discover global and scoped enum declaration files. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover global and scoped enum declaration files. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None, isolate_declaration_kind=false))] pub fn discover_enum_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, isolate_declaration_kind: bool, ) -> PyResult> { - declaration_records::discover_enum_files(project_dir, on_fault, isolate_declaration_kind) + discovered( + &faults(project_dir, on_fault), + CollectionKind::Enums, + isolate_declaration_kind, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_macro_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_macro_files.rs index fbd7a0a5da..55bac8b128 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_macro_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_macro_files.rs @@ -1,16 +1,22 @@ //! Discover global and scoped project macro Python files. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover global and scoped project macro Python files. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, isolate_declaration_kind=false))] pub fn discover_macro_files<'py>( project_dir: &Bound<'py, PyAny>, isolate_declaration_kind: bool, ) -> PyResult> { - declaration_records::discover_macro_files(project_dir, isolate_declaration_kind) + discovered( + &faults(project_dir, None), + CollectionKind::Macros, + isolate_declaration_kind, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_files.rs index 2ac5e14538..d2b189514b 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_files.rs @@ -1,17 +1,45 @@ //! Discover the (selected) model files natively, reporting failing files to `on_fault`. -use crate::discovery::_helpers::records::model_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use crate::discovery::main::records::discovered_model_files::discovered_model_files; +use crate::discovery::models::ModelFileSelection; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_python_base::boundary::models::KeywordArguments; /// Discover the (selected) model files natively, reporting failing files to `on_fault`. -#[inline] +#[pyfunction] +#[pyo3(signature = (**options))] pub fn discover_model_files<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - model_records::discover_model_files(py, options) + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "discover_model_files", + options, + ( + &[ + "project_dir", + "extract_implicit_alias_columns", + "extract_output_column_locations", + ], + &["selected_model_names", "on_fault"], + ), + )?; + discovered_model_files( + &arguments.value("project_dir")?, + ( + arguments + .value("extract_implicit_alias_columns")? + .extract()?, + arguments + .value("extract_output_column_locations")? + .extract()?, + ), + ModelFileSelection { + selected_model_names: Some(arguments.value("selected_model_names")?), + on_fault: Some(arguments.value("on_fault")?), + }, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_schema_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_schema_files.rs index 9c886cd419..be85d1fae1 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_schema_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_model_schema_files.rs @@ -1,16 +1,22 @@ //! Discover project-wide and scoped reusable model schemas. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover project-wide and scoped reusable model schemas. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] pub fn discover_model_schema_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, ) -> PyResult> { - declaration_records::discover_model_schema_files(project_dir, on_fault) + discovered( + &faults(project_dir, on_fault), + CollectionKind::ModelSchemas, + false, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_scenario_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_scenario_files.rs index 1dd47a45dc..517373e7a2 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_scenario_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_scenario_files.rs @@ -1,16 +1,44 @@ //! Discover scenario files natively, reporting failing files to `on_fault`. -use crate::discovery::_helpers::records::sql_test_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::faults::FileFaults; +use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::records::sql_test_records::discovered_files; +use crate::discovery::_helpers::records::sql_test_records::path_class; +use crate::discovery::_helpers::records::sql_test_records::records; +use crate::discovery::_helpers::records::sql_test_records::scenario_file; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::models::FileOutcome; +use sqlbuild_discovery::sql_tests::main::discover_scenario_files::discover_scenario_files as discover_scenarios; +use sqlbuild_discovery::sql_tests::models::DiscoveredScenarioFile; +use sqlbuild_discovery::tree::models::ProjectTree; /// Discover scenario files natively, reporting failing files to `on_fault`. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] pub fn discover_scenario_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, ) -> PyResult> { - sql_test_records::discover_scenario_files(project_dir, on_fault) + let py: Python<'py> = project_dir.py(); + let (tree, files) = discovered_files(project_dir, discover_scenarios)?; + let tree: &ProjectTree = &tree.get().inner; + let kept: Vec<(Bound<'py, PyAny>, FileOutcome)> = files + .into_iter() + .map(|file| { + Ok(( + path_class(py)?.call1((path_object(py, tree, file.relative_path)?,))?, + file.outcome, + )) + }) + .collect::>()?; + records( + &FileFaults { + project_dir: project_dir.clone(), + on_fault, + }, + kept, + scenario_file, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_schema_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_schema_files.rs index 3481d9f5bd..b1abfbd1c7 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_schema_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_schema_files.rs @@ -1,15 +1,79 @@ //! Discover model schema.yml and seed declaration files natively. -use crate::discovery::_helpers::records::yaml_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyTuple; +use crate::discovery::_helpers::failures::faults::record_tuple; +use crate::discovery::_helpers::failures::raised::{discovery_error, stage_error}; +use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::records::yaml_records::ListingTree; +use crate::discovery::_helpers::records::yaml_records::SCHEMA_KIND; +use crate::discovery::_helpers::records::yaml_records::YamlKind; +use crate::discovery::_helpers::records::yaml_records::listing_tree; +use crate::discovery::_helpers::records::yaml_records::load_files; +use crate::discovery::_helpers::records::yaml_records::path_class; +use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; +use crate::discovery::_helpers::sources::yaml_entries::parse_schema_document; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyStringMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction}; +use sqlbuild_discovery::yaml_files::main::schema_yaml_paths::schema_yaml_paths; +use sqlbuild_discovery::yaml_files::models::YamlPathFailure; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; /// Discover model schema.yml and seed declaration files natively. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir))] pub fn discover_schema_files<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { - yaml_records::discover_schema_files(project_dir) + let py: Python<'py> = project_dir.py(); + in_snapshot(project_dir, || { + let listing: ListingTree<'py> = listing_tree(project_dir)?; + let paths: Vec = match schema_yaml_paths(listing.tree()) { + Ok(paths) => paths, + Err(YamlPathFailure::Stage(failure)) => { + return Err(stage_error(listing.tree(), project_dir, failure)); + } + Err(YamlPathFailure::YamlSeedDeclaration(path)) => { + let relative: Bound<'py, PyAny> = + path_class(py)?.call1((path_object(py, listing.tree(), path)?,))?; + return Err(discovery_error( + py, + SCHEMA_KIND, + format!( + "Seed declaration file {} must use .yml; .yaml is not supported", + relative.str()? + ), + None, + )); + } + }; + if paths.is_empty() { + return record_tuple(py, Vec::new()); + } + let mut records: Vec> = Vec::new(); + load_files( + project_dir, + (paths, YamlKind::Schema), + |(relative_path, file_path), built| { + let (contents, loaded) = built?; + let shown: String = file_path.str()?.to_str()?.to_owned(); + records.push(new_record_with( + py, + RecordId::DiscoveredSchemaFile, + &[ + ("file_path", file_path.clone()), + ("relative_path", relative_path), + ("contents", contents.into_bound_py_any(py)?), + ("model_entries", PyTuple::empty(py).into_any()), + ( + "seed_entries", + parse_schema_document(py, loaded, &shown)?.into_any(), + ), + ], + )?); + Ok(()) + }, + )?; + record_tuple(py, records) + }) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_seed_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_seed_files.rs index 89c3cc5a60..06db4657df 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_seed_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_seed_files.rs @@ -1,13 +1,15 @@ //! Discover seed CSV files under seeds/. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover seed CSV files under seeds/. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir))] pub fn discover_seed_files<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { - declaration_records::discover_seed_files(project_dir) + discovered(&faults(project_dir, None), CollectionKind::Seeds, false) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_source_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_source_files.rs index bd3e3d36ef..cbdb40e600 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_source_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_source_files.rs @@ -1,16 +1,64 @@ //! Discover source YAML files natively, reporting failing files to `on_fault`. -use crate::discovery::_helpers::records::yaml_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyTuple; +use crate::discovery::_helpers::failures::faults::{FileFaults, record_tuple}; +use crate::discovery::_helpers::failures::raised::stage_error; +use crate::discovery::_helpers::records::yaml_records::ListingTree; +use crate::discovery::_helpers::records::yaml_records::YamlKind; +use crate::discovery::_helpers::records::yaml_records::listing_tree; +use crate::discovery::_helpers::records::yaml_records::load_files; +use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; +use crate::discovery::_helpers::sources::yaml_entries::parse_sources_document; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyStringMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction}; +use sqlbuild_discovery::yaml_files::main::source_yaml_paths::source_yaml_paths; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; /// Discover source YAML files natively, reporting failing files to `on_fault`. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] pub fn discover_source_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, ) -> PyResult> { - yaml_records::discover_source_files(project_dir, on_fault) + let py: Python<'py> = project_dir.py(); + in_snapshot(project_dir, || { + let listing: ListingTree<'py> = listing_tree(project_dir)?; + let paths: Vec = source_yaml_paths(listing.tree()) + .map_err(|failure| stage_error(listing.tree(), project_dir, failure))?; + if paths.is_empty() { + return record_tuple(py, Vec::new()); + } + let faults = FileFaults { + project_dir: project_dir.clone(), + on_fault, + }; + let mut records: Vec> = Vec::new(); + load_files( + project_dir, + (paths, YamlKind::Source), + |(relative_path, file_path), built| { + let record = built.and_then(|(contents, loaded)| { + let shown: String = file_path.str()?.to_str()?.to_owned(); + new_record_with( + py, + RecordId::DiscoveredSourceFile, + &[ + ("file_path", file_path.clone()), + ("relative_path", relative_path.clone()), + ("contents", contents.into_bound_py_any(py)?), + ( + "source_entries", + parse_sources_document(py, loaded, &shown)?.into_any(), + ), + ], + ) + }); + records.extend(faults.kept(&relative_path, record)?); + Ok(()) + }, + )?; + record_tuple(py, records) + }) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_function_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_function_files.rs index 2cb6438195..49665dc655 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_function_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_function_files.rs @@ -1,16 +1,22 @@ //! Discover SQL function files under functions/sql/. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover SQL function files under functions/sql/. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] pub fn discover_sql_function_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, ) -> PyResult> { - declaration_records::discover_sql_function_files(project_dir, on_fault) + discovered( + &faults(project_dir, on_fault), + CollectionKind::SqlFunctions, + false, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_hook_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_hook_files.rs index 6d970cbb61..94416431aa 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_hook_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_sql_hook_files.rs @@ -1,16 +1,22 @@ //! Discover named SQL lifecycle hooks in their project-wide and scoped roles. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::discovered; +use crate::discovery::_helpers::records::declaration_records::faults; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::declaration_files::models::CollectionKind; /// Discover named SQL lifecycle hooks in their project-wide and scoped roles. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, on_fault=None))] pub fn discover_sql_hook_files<'py>( project_dir: &Bound<'py, PyAny>, on_fault: Option>, ) -> PyResult> { - declaration_records::discover_sql_hook_files(project_dir, on_fault) + discovered( + &faults(project_dir, on_fault), + CollectionKind::SqlHooks, + false, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs index 3f73ba9034..69ef1a190f 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs @@ -1,17 +1,46 @@ //! Discover the (selected) SQL test files natively, reporting failing files to `on_fault`. -use crate::discovery::_helpers::records::sql_test_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::faults::FileFaults; +use crate::discovery::_helpers::project::project_files::path_object; +use crate::discovery::_helpers::records::sql_test_records::discovered_files; +use crate::discovery::_helpers::records::sql_test_records::path_class; +use crate::discovery::_helpers::records::sql_test_records::records; +use crate::discovery::_helpers::records::sql_test_records::test_file; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyTuple; +use sqlbuild_discovery::models::FileOutcome; +use sqlbuild_discovery::sql_tests::main::discover_sql_test_files::discover_sql_test_files as discover_tests; +use sqlbuild_discovery::sql_tests::models::DiscoveredSqlTestFile; +use sqlbuild_discovery::tree::models::ProjectTree; /// Discover the (selected) SQL test files natively, reporting failing files to `on_fault`. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir, selected_paths=None, on_fault=None))] pub fn discover_test_files<'py>( project_dir: &Bound<'py, PyAny>, selected_paths: Option>, on_fault: Option>, ) -> PyResult> { - sql_test_records::discover_test_files(project_dir, selected_paths, on_fault) + let py: Python<'py> = project_dir.py(); + let (tree, files) = discovered_files(project_dir, discover_tests)?; + let tree: &ProjectTree = &tree.get().inner; + let mut kept: Vec<(Bound<'py, PyAny>, FileOutcome)> = Vec::new(); + for file in files { + let relative: Bound<'py, PyAny> = path_object(py, tree, file.relative_path)?.into_bound(py); + if let Some(selected) = &selected_paths + && !selected.contains(project_dir.div(&relative)?.call_method0("resolve")?)? + { + continue; + } + kept.push((path_class(py)?.call1((relative,))?, file.outcome)); + } + records( + &FileFaults { + project_dir: project_dir.clone(), + on_fault, + }, + kept, + test_file, + ) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/discovered_model_files.rs b/crates/sqlbuild-python-front/src/discovery/main/records/discovered_model_files.rs index 9df1381eaf..8e9439d82b 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/discovered_model_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/discovered_model_files.rs @@ -1,18 +1,47 @@ //! The (selected) model files; `extraction` is `(implicit alias columns, output locations)`. -use crate::discovery::_helpers::records::model_records; +use crate::discovery::_helpers::failures::faults::{FileFaults, record_tuple}; +use crate::discovery::_helpers::records::model_records::ModelItem; +use crate::discovery::_helpers::records::model_records::model_items; use crate::discovery::models::ModelFileSelection; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; +use sqlbuild_discovery::model_files::main::model_file_options::model_file_options; +use sqlbuild_discovery::model_files::models::ModelFileOptions; /// The (selected) model files; `extraction` is `(implicit alias columns, output locations)`. -#[inline] pub fn discovered_model_files<'py>( project_dir: &Bound<'py, PyAny>, extraction: (bool, bool), selection: ModelFileSelection<'py>, ) -> PyResult> { - model_records::discovered_model_files(project_dir, extraction, selection) + let py: Python<'py> = project_dir.py(); + let options: ModelFileOptions = model_file_options( + extraction, + sqlbuild_python_base::compile_context::main::delta::injection::injection() + .map(|injection| std::sync::Arc::new(injection.model_paths.clone())), + ); + let items: Vec> = model_items(project_dir, &options)?; + let faults = FileFaults { + project_dir: project_dir.clone(), + on_fault: selection.on_fault.filter(|on_fault| !on_fault.is_none()), + }; + let selected: Option> = selection + .selected_model_names + .filter(|selected| !selected.is_none()); + let mut records: Vec> = Vec::with_capacity(items.len()); + for (relative_path, built) in items { + if let Some(selected) = &selected + && !selected.contains(relative_path.name(py)?)? + { + continue; + } + match built { + Ok(record) => records.push(record), + Err(error) => { + records.extend(faults.kept(&relative_path.object(py)?, Err(error))?); + } + } + } + record_tuple(py, records) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/externally_described_models.rs b/crates/sqlbuild-python-front/src/discovery/main/records/externally_described_models.rs index fd6cd14803..c222efacc5 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/externally_described_models.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/externally_described_models.rs @@ -1,15 +1,17 @@ //! Model files that `[defaults]`, `[path_defaults]` or a model schema describe. -use crate::discovery::_helpers::records::description_inputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyFrozenSet; +use crate::discovery::_helpers::records::description_inputs::discover_model_description_inputs; +use crate::model_config::main::model_nodes::externally_described_models::externally_described_models as described_models; +use crate::render::main::declarations::public_model_schema_index::public_model_schema_index; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet}; /// Model files that `[defaults]`, `[path_defaults]` or a model schema describe. -#[inline] pub fn externally_described_models<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { - description_inputs::externally_described_models(project_dir) + let py: Python<'py> = project_dir.py(); + let inputs: Bound<'py, PyAny> = discover_model_description_inputs(project_dir)?; + let schemas: Bound<'py, PyDict> = public_model_schema_index(inputs.clone())?; + PyFrozenSet::new(py, described_models(inputs, schemas)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/parse_sql_test_file.rs b/crates/sqlbuild-python-front/src/discovery/main/records/parse_sql_test_file.rs index c10bd475da..26e80582f5 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/parse_sql_test_file.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/parse_sql_test_file.rs @@ -1,16 +1,25 @@ //! Parse in-memory SQL test contents into ordered blocks, as test discovery does. -use crate::discovery::_helpers::records::sql_test_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyTuple; +use crate::discovery::_helpers::failures::raised::failure_error; +use crate::discovery::_helpers::records::sql_test_records::options; +use crate::discovery::_helpers::records::sql_test_records::test_blocks; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyStringMethods, PyTuple}; +use sqlbuild_discovery::sql_tests::main::parse_sql_test_text::parse_sql_test_text; +use sqlbuild_discovery::sql_tests::models::{DiscoveredSqlTestFile, SqlTestFileOptions}; /// Parse in-memory SQL test contents into ordered blocks, as test discovery does. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, contents, file_path))] pub fn parse_sql_test_file<'py>( contents: String, file_path: &Bound<'py, PyAny>, ) -> PyResult> { - sql_test_records::parse_sql_test_file(contents, file_path) + let py: Python<'py> = file_path.py(); + let path: String = file_path.str()?.to_str()?.to_owned(); + let options: SqlTestFileOptions = options()?; + let parsed = py.detach(|| parse_sql_test_text(&path, contents, &options)); + let file: DiscoveredSqlTestFile = parsed.map_err(|failure| failure_error(py, failure))?; + test_blocks(file_path, file.blocks, file.failure) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/projected_text.rs b/crates/sqlbuild-python-front/src/discovery/main/records/projected_text.rs index 2c1eadeda9..d420097c19 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/projected_text.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/projected_text.rs @@ -1,18 +1,17 @@ //! `text` as the Python string `cache` shares with every reader, built on first read. -use crate::discovery::_helpers::records::model_file_record; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, Py, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::PyString; /// `text` as the Python string `cache` shares with every reader, built on first read. -#[inline] pub fn projected_text<'py>( py: Python<'py>, cache: &PyOnceLock>, text: &str, ) -> Bound<'py, PyString> { - model_file_record::projected_text(py, cache, text) + cache + .get_or_init(py, || PyString::new(py, text).unbind()) + .bind(py) + .clone() } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/register_declaration_records.rs b/crates/sqlbuild-python-front/src/discovery/main/records/register_declaration_records.rs index 19486f7f7c..a18ff416eb 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/register_declaration_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/register_declaration_records.rs @@ -1,12 +1,29 @@ //! Register the declaration records functions and classes. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::main::records::discover_adapter_file::discover_adapter_file; +use crate::discovery::main::records::discover_audit_files::discover_audit_files; +use crate::discovery::main::records::discover_constant_files::discover_constant_files; +use crate::discovery::main::records::discover_enum_files::discover_enum_files; +use crate::discovery::main::records::discover_macro_files::discover_macro_files; +use crate::discovery::main::records::discover_model_schema_files::discover_model_schema_files; +use crate::discovery::main::records::discover_seed_files::discover_seed_files; +use crate::discovery::main::records::discover_sql_function_files::discover_sql_function_files; +use crate::discovery::main::records::discover_sql_hook_files::discover_sql_hook_files; +use crate::discovery::main::records::retained_discovery_session::retained_discovery_session; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the declaration records functions and classes. -#[inline] pub fn register_declaration_records(module: &Bound<'_, PyModule>) -> PyResult<()> { - declaration_records::register(module) + module.add_function(wrap_pyfunction!(discover_enum_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_constant_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_model_schema_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_sql_function_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_sql_hook_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_audit_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_macro_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_seed_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_adapter_file, module)?)?; + module.add_function(wrap_pyfunction!(retained_discovery_session, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/register_description_inputs.rs b/crates/sqlbuild-python-front/src/discovery/main/records/register_description_inputs.rs index f633072057..a0e2c138eb 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/register_description_inputs.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/register_description_inputs.rs @@ -1,12 +1,11 @@ //! Register the description inputs functions and classes. -use crate::discovery::_helpers::records::description_inputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::description_inputs::discover_model_description_inputs; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the description inputs functions and classes. -#[inline] pub fn register_description_inputs(module: &Bound<'_, PyModule>) -> PyResult<()> { - description_inputs::register(module) + module.add_function(wrap_pyfunction!(discover_model_description_inputs, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/register_model_records.rs b/crates/sqlbuild-python-front/src/discovery/main/records/register_model_records.rs index 1ca1846266..1daae8294a 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/register_model_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/register_model_records.rs @@ -1,12 +1,10 @@ //! Register the model records functions and classes. -use crate::discovery::_helpers::records::model_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::main::records::discover_model_files::discover_model_files; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the model records functions and classes. -#[inline] pub fn register_model_records(module: &Bound<'_, PyModule>) -> PyResult<()> { - model_records::register(module) + module.add_function(wrap_pyfunction!(discover_model_files, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs b/crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs index ab0033786d..ce02a5a2f9 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs @@ -1,12 +1,16 @@ //! Register the sql test records functions and classes. -use crate::discovery::_helpers::records::sql_test_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::sql_test_records::parse_sql_scenario_file; +use crate::discovery::main::records::discover_scenario_files::discover_scenario_files; +use crate::discovery::main::records::discover_test_files::discover_test_files; +use crate::discovery::main::records::parse_sql_test_file::parse_sql_test_file; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the sql test records functions and classes. -#[inline] pub fn register_sql_test_records(module: &Bound<'_, PyModule>) -> PyResult<()> { - sql_test_records::register(module) + module.add_function(wrap_pyfunction!(discover_test_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_scenario_files, module)?)?; + module.add_function(wrap_pyfunction!(parse_sql_test_file, module)?)?; + module.add_function(wrap_pyfunction!(parse_sql_scenario_file, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs b/crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs index 9f117e1b44..ce8112b7c9 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs @@ -1,12 +1,16 @@ //! Register the yaml records functions and classes. -use crate::discovery::_helpers::records::yaml_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::yaml_records::load_yaml_document; +use crate::discovery::_helpers::records::yaml_records::load_yaml_files; +use crate::discovery::main::records::discover_schema_files::discover_schema_files; +use crate::discovery::main::records::discover_source_files::discover_source_files; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the yaml records functions and classes. -#[inline] pub fn register_yaml_records(module: &Bound<'_, PyModule>) -> PyResult<()> { - yaml_records::register(module) + module.add_function(wrap_pyfunction!(discover_source_files, module)?)?; + module.add_function(wrap_pyfunction!(discover_schema_files, module)?)?; + module.add_function(wrap_pyfunction!(load_yaml_files, module)?)?; + module.add_function(wrap_pyfunction!(load_yaml_document, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/records/retained_discovery_session.rs b/crates/sqlbuild-python-front/src/discovery/main/records/retained_discovery_session.rs index 75b0fd3daf..5c6ea9667e 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/records/retained_discovery_session.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/records/retained_discovery_session.rs @@ -1,14 +1,21 @@ //! The session this discovery pass read its declaration files with, if any. -use crate::discovery::_helpers::records::declaration_records; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::records::declaration_records::SESSION_MEMO_KEY; +use crate::discovery::_helpers::snapshots::native_classes::NativeDiscoverySession; +use crate::discovery::_helpers::snapshots::snapshot::snapshot_memo; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDictMethods; /// The session this discovery pass read its declaration files with, if any. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir))] pub fn retained_discovery_session<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { - declaration_records::retained_discovery_session(project_dir) + let py: Python<'py> = project_dir.py(); + Ok(snapshot_memo(project_dir)? + .get_item(SESSION_MEMO_KEY)? + .filter(|session| session.is_instance_of::()) + .unwrap_or_else(|| py.None().into_bound(py))) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/directory_snapshot_scope.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/directory_snapshot_scope.rs index 611b9a2b24..54342c1938 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/snapshots/directory_snapshot_scope.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/directory_snapshot_scope.rs @@ -1,12 +1,15 @@ //! Share one directory snapshot across every discovery call for `project_dir` inside the scope. -use crate::discovery::_helpers::snapshots::snapshot; use crate::discovery::models::NativeDirectorySnapshotScope; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; +use pyo3::prelude::{Bound, PyAny}; +use pyo3::pyfunction; /// Share one directory snapshot across every discovery call for `project_dir` inside the scope. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir))] pub fn directory_snapshot_scope(project_dir: Bound<'_, PyAny>) -> NativeDirectorySnapshotScope { - snapshot::directory_snapshot_scope(project_dir) + NativeDirectorySnapshotScope { + project_dir: project_dir.unbind(), + token: None, + } } diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/discover_scope_snapshot.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/discover_scope_snapshot.rs index bbd45acb50..41052eb99f 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/snapshots/discover_scope_snapshot.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/discover_scope_snapshot.rs @@ -1,14 +1,15 @@ //! Parse the scope-relevant authored roots while retaining independent file faults. -use crate::discovery::_helpers::snapshots::scope_snapshot; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::snapshots::scope_snapshot::tolerant_discovery; +use crate::discovery::_helpers::snapshots::snapshot::in_snapshot; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; /// Parse the scope-relevant authored roots while retaining independent file faults. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, project_dir))] pub fn discover_scope_snapshot<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { - scope_snapshot::discover_scope_snapshot(project_dir) + in_snapshot(project_dir, || tolerant_discovery(project_dir)) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/escaped.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/escaped.rs index 860c66ec23..7dbe88f293 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/snapshots/escaped.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/escaped.rs @@ -1,12 +1,25 @@ //! Python's `_escaped`: undecodable name bytes as `\xNN`, other lone surrogates as `\uXXXX`. -use crate::discovery::_helpers::snapshots::snapshot; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::snapshots::snapshot::SURROGATE_PASS; +use crate::discovery::_helpers::snapshots::snapshot::UTF16_ENCODING; +use crate::discovery::_helpers::snapshots::snapshot::escaped_unit; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult}; use pyo3::types::PyString; /// Python's `_escaped`: undecodable name bytes as `\xNN`, other lone surrogates as `\uXXXX`. -#[inline] pub fn escaped(text: &Bound<'_, PyString>) -> PyResult { - snapshot::escaped(text) + let encoded: Vec = text + .call_method1("encode", (UTF16_ENCODING, SURROGATE_PASS))? + .extract()?; + let units = encoded + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u16::from_le_bytes(*pair)); + Ok(char::decode_utf16(units) + .map(|decoded| match decoded { + Ok(character) => character.to_string(), + Err(error) => escaped_unit(error.unpaired_surrogate()), + }) + .collect()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/in_snapshot.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/in_snapshot.rs index c21854391c..6205158bad 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/snapshots/in_snapshot.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/in_snapshot.rs @@ -1,9 +1,7 @@ //! Run `body` with one snapshot of `project_dir` shared by every discovery call inside it. use crate::discovery::_helpers::snapshots::snapshot; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Run `body` with one snapshot of `project_dir` shared by every discovery call inside it. #[inline] diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_scope_snapshot.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_scope_snapshot.rs index de088e3cb1..230e013c51 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_scope_snapshot.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_scope_snapshot.rs @@ -1,12 +1,10 @@ //! Register the scope snapshot functions and classes. -use crate::discovery::_helpers::snapshots::scope_snapshot; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::main::snapshots::discover_scope_snapshot::discover_scope_snapshot; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scope snapshot functions and classes. -#[inline] pub fn register_scope_snapshot(module: &Bound<'_, PyModule>) -> PyResult<()> { - scope_snapshot::register(module) + module.add_function(wrap_pyfunction!(discover_scope_snapshot, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs b/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs index e29fc29827..6976e44a1f 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs @@ -1,12 +1,15 @@ //! Register the snapshot functions and classes. -use crate::discovery::_helpers::snapshots::snapshot; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::snapshots::native_classes::NativeDirectorySnapshot; +use crate::discovery::main::snapshots::directory_snapshot_scope::directory_snapshot_scope; +use crate::discovery::models::NativeDirectorySnapshotScope; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the snapshot functions and classes. -#[inline] pub fn register_snapshot(module: &Bound<'_, PyModule>) -> PyResult<()> { - snapshot::register(module) + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(directory_snapshot_scope, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/consumer_uses.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/consumer_uses.rs index bf17b61bf6..b84aaf6c0b 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/consumer_uses.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/consumer_uses.rs @@ -1,12 +1,19 @@ //! `(provider, consumer)` for each provider the named consumers, such as checks or loaders, inject. -use crate::discovery::_helpers::sources::provider_secrets; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::main::sources::provider_uses::provider_uses; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field; /// `(provider, consumer)` for each provider the named consumers, such as checks or loaders, inject. -#[inline] pub fn consumer_uses(consumers: &Bound<'_, PyAny>) -> PyResult> { - provider_secrets::consumer_uses(consumers) + let mut uses: Vec<(String, String)> = Vec::new(); + for consumer in consumers.try_iter()? { + let consumer: Bound<'_, PyAny> = consumer?; + let name: String = field(&consumer, "name")?.extract()?; + uses.extend(provider_uses( + &field(&consumer, "provider_usages")?, + Some(&name), + )?); + } + Ok(uses) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/invocation_allows_missing_secrets.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/invocation_allows_missing_secrets.rs index 95d0bc0f90..f9ee98d687 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/invocation_allows_missing_secrets.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/invocation_allows_missing_secrets.rs @@ -1,9 +1,8 @@ //! Whether the running invocation passed `--allow-missing-secrets`. -use crate::discovery::_helpers::sources::provider_secrets; +use crate::discovery::_helpers::sources::provider_secrets::INVOCATION_ALLOWS_MISSING_SECRETS; /// Whether the running invocation passed `--allow-missing-secrets`. -#[inline] pub fn invocation_allows_missing_secrets() -> bool { - provider_secrets::invocation_allows_missing_secrets() + INVOCATION_ALLOWS_MISSING_SECRETS.get() } diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs index 915c41e145..3f64b315b8 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs @@ -1,15 +1,25 @@ //! Every job, job group, calendar and sensor of `project_dir`, or `OrchestrationDeclarationError` (D017). -use crate::discovery::_helpers::sources::orchestration_files; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::failures::raised::failure_error; +use crate::discovery::_helpers::sources::orchestration_files::orchestration_error; +use crate::discovery::_helpers::sources::orchestration_values::check_orchestration_values; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_discovery::orchestration_files::main::orchestration_inventory::orchestration_inventory; use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; +use sqlbuild_discovery::tree::models::ProjectTree; +use std::path::PathBuf; /// Every job, job group, calendar and sensor of `project_dir`, or `OrchestrationDeclarationError` (D017). -#[inline] pub fn orchestration_declarations( project_dir: &Bound<'_, PyAny>, ) -> PyResult { - orchestration_files::orchestration_declarations(project_dir) + let py: Python<'_> = project_dir.py(); + let root: PathBuf = project_dir.extract()?; + let tree = ProjectTree::new(&root); + let version = py.version_info(); + let inventory = py.detach(|| orchestration_inventory(&tree, (version.major, version.minor))); + let inventory = + inventory.map_err(|failure| orchestration_error(&tree, project_dir, failure))?; + check_orchestration_values(py, &inventory).map_err(|failure| failure_error(py, failure))?; + Ok(inventory) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/provider_uses.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/provider_uses.rs index 872118c52d..70ce058b68 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/provider_uses.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/provider_uses.rs @@ -1,15 +1,15 @@ //! `(provider, consumer)` for each usage, named by the usage's own `consumer_name` or else by `consumer`. -use crate::discovery::_helpers::sources::provider_secrets; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::sources::provider_secrets::usage_pair; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// `(provider, consumer)` for each usage, named by the usage's own `consumer_name` or else by `consumer`. -#[inline] pub fn provider_uses( usages: &Bound<'_, PyAny>, consumer: Option<&str>, ) -> PyResult> { - provider_secrets::provider_uses(usages, consumer) + usages + .try_iter()? + .map(|usage| usage_pair(&usage?, consumer)) + .collect() } diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/register_declaration_files.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/register_declaration_files.rs index 407e5eb323..ea23bafb31 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/register_declaration_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/register_declaration_files.rs @@ -1,12 +1,13 @@ //! Register the declaration files functions and classes. -use crate::discovery::_helpers::sources::declaration_files; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::snapshots::native_classes::NativeDiscoverySession; +use crate::discovery::_helpers::sources::declaration_files::parse_declaration_contents; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the declaration files functions and classes. -#[inline] pub fn register_declaration_files(module: &Bound<'_, PyModule>) -> PyResult<()> { - declaration_files::register(module) + module.add_class::()?; + module.add_function(wrap_pyfunction!(parse_declaration_contents, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/register_yaml_entries.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/register_yaml_entries.rs index 37f5ed9420..325ad3623f 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/register_yaml_entries.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/register_yaml_entries.rs @@ -1,12 +1,12 @@ //! Register the yaml entries functions and classes. -use crate::discovery::_helpers::sources::yaml_entries; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::sources::yaml_entries::parse_schema_document; +use crate::discovery::_helpers::sources::yaml_entries::parse_sources_document; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the yaml entries functions and classes. -#[inline] pub fn register_yaml_entries(module: &Bound<'_, PyModule>) -> PyResult<()> { - yaml_entries::register(module) + module.add_function(wrap_pyfunction!(parse_schema_document, module)?)?; + module.add_function(wrap_pyfunction!(parse_sources_document, module)?) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/require_used_secrets.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/require_used_secrets.rs index 7342258da8..652eafdb1e 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/require_used_secrets.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/require_used_secrets.rs @@ -1,9 +1,7 @@ //! Fail before anything runs when a provider in `uses` lacks required secrets, listing each with what needs it. use crate::discovery::_helpers::sources::provider_secrets; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Fail before anything runs when a provider in `uses` lacks required secrets, listing each with what needs it. #[inline] diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/sink_uses.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/sink_uses.rs index 5d4fc617cf..295e5b35ef 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/sink_uses.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/sink_uses.rs @@ -1,12 +1,16 @@ //! The providers the project's lifecycle and command-output sinks use, which run with every command. -use crate::discovery::_helpers::sources::provider_secrets; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::discovery::_helpers::sources::provider_secrets::SINK_COLLECTIONS; +use crate::discovery::main::sources::consumer_uses::consumer_uses; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// The providers the project's lifecycle and command-output sinks use, which run with every command. -#[inline] pub fn sink_uses(discovered: &Bound<'_, PyAny>) -> PyResult> { - provider_secrets::sink_uses(discovered) + let mut uses: Vec<(String, String)> = Vec::new(); + for collection in SINK_COLLECTIONS { + if let Some(sinks) = discovered.getattr_opt(collection)? { + uses.extend(consumer_uses(&sinks)?); + } + } + Ok(uses) } diff --git a/crates/sqlbuild-python-front/src/discovery/models.rs b/crates/sqlbuild-python-front/src/discovery/models.rs index df8f05bc3b..bbe037f371 100644 --- a/crates/sqlbuild-python-front/src/discovery/models.rs +++ b/crates/sqlbuild-python-front/src/discovery/models.rs @@ -6,8 +6,8 @@ use crate::discovery::_helpers::records::model_file_record::HeaderValues; use crate::discovery::_helpers::records::model_file_record::Locations; use crate::discovery::_helpers::records::model_file_record::ModelFileFields; use crate::discovery::_helpers::records::model_file_record::ModelFilePaths; -use crate::discovery::_helpers::records::model_file_record::projected_text; use crate::discovery::_helpers::sources::provider_secrets::INVOCATION_ALLOWS_MISSING_SECRETS; +use crate::discovery::main::records::projected_text::projected_text; use pyo3::FromPyObject; use pyo3::IntoPyObjectExt; use pyo3::exceptions::PyTypeError; diff --git a/crates/sqlbuild-python-front/src/macro_bridge/_helpers/first_compile.rs b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/first_compile.rs index 11e0d1c083..c80df95845 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/_helpers/first_compile.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/first_compile.rs @@ -2,24 +2,23 @@ use std::sync::{Mutex, MutexGuard, PoisonError}; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::pyfunction; use sqlbuild_python_base::compile_context::main::delta::injection::injection; /// Whether a compile claimed the first-compile slot, and whether a delta compile only borrowed it. #[derive(Debug)] -struct FirstCompile { +pub(crate) struct FirstCompile { claimed: bool, - lent_to_delta: bool, + pub(crate) lent_to_delta: bool, } -static FIRST_COMPILE: Mutex = Mutex::new(FirstCompile { +pub(crate) static FIRST_COMPILE: Mutex = Mutex::new(FirstCompile { claimed: false, lent_to_delta: false, }); -fn locked(slot: &Mutex) -> MutexGuard<'_, T> { +pub(crate) fn locked(slot: &Mutex) -> MutexGuard<'_, T> { slot.lock().unwrap_or_else(PoisonError::into_inner) } @@ -41,12 +40,3 @@ pub(crate) fn claim_first_compile() -> bool { first.claimed = true; granted } - -/// Keep the delta compile's claim: it finished, so no full compile of this invocation follows. -pub(crate) fn keep_delta_claim() { - locked(&FIRST_COMPILE).lent_to_delta = false; -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(claim_first_compile, module)?) -} diff --git a/crates/sqlbuild-python-front/src/macro_bridge/_helpers/module_digests.rs b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/module_digests.rs index 45cc26a97a..a593eabe83 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/_helpers/module_digests.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/module_digests.rs @@ -1,16 +1,16 @@ //! The module file digests a macro call store records and checks before reuse. +use crate::macro_bridge::main::text::text; use std::path::PathBuf; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyList, PyListMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cache::module_digests::main::module_digests_metadata::module_digests_metadata as metadata_text; use sqlbuild_core::json::models::JsonValue; -use crate::macro_bridge::_helpers::store_keys::text; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::cache::main::digest_files::digest_files; use sqlbuild_python_base::frontier::main::emit_error::emit_error; @@ -48,7 +48,7 @@ pub(crate) fn digest_module_files<'py>( /// Encode module file digests for the store's metadata. #[pyfunction] -fn module_digests_metadata<'py>( +pub(crate) fn module_digests_metadata<'py>( py: Python<'py>, digests: &Bound<'py, PyAny>, ) -> PyResult> { @@ -83,7 +83,7 @@ fn digest_row_check(row: &Bound<'_, PyAny>) -> PyResult { /// Return the stored module digests when every module file still has them, else None. #[pyfunction] #[pyo3(signature = (*, metadata, known))] -fn unchanged_module_digests<'py>( +pub(crate) fn unchanged_module_digests<'py>( py: Python<'py>, metadata: &Bound<'py, PyAny>, known: &Bound<'py, PyAny>, @@ -125,9 +125,3 @@ fn unchanged_module_digests<'py>( None }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(digest_module_files, module)?)?; - module.add_function(wrap_pyfunction!(module_digests_metadata, module)?)?; - module.add_function(wrap_pyfunction!(unchanged_module_digests, module)?) -} diff --git a/crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs index 21573a9fe5..49b76e2977 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs @@ -1,24 +1,18 @@ //! Value-based text of everything a stored macro call result depends on besides its call text. +use crate::macro_bridge::main::text::text; +use pyo3::create_exception; use pyo3::exceptions::{PyRecursionError, PyUnicodeEncodeError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{ PyBool, PyBytes, PyDict, PyFloat, PyFrozenSet, PyInt, PyList, PySet, PyString, PyStringMethods, PyTuple, PyTypeMethods, }; -use pyo3::{create_exception, pyfunction, wrap_pyfunction}; -use sqlbuild_python_base::boundary::main::utf16_units::utf16_units; use sqlbuild_cache::macro_store_keys::main::canonical_sort_text::canonical_sort_text; -use sqlbuild_cache::macro_store_keys::main::context_store_token::context_store_token as context_key; -use sqlbuild_cache::macro_store_keys::main::macro_store_token::macro_store_token as macro_key; use sqlbuild_core::json::models::{JsonInteger, JsonValue}; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::frontier::main::emit_error::emit_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; @@ -33,7 +27,7 @@ const CANONICAL_STACK_GROWTH: usize = 2 * 1024 * 1024; create_exception!(_native, UncacheableValue, pyo3::exceptions::PyException); /// Whether `error` is one that makes a value unkeyable rather than a failure to report. -fn unstable(py: Python<'_>, error: &PyErr) -> bool { +pub(crate) fn unstable(py: Python<'_>, error: &PyErr) -> bool { error.is_instance_of::(py) || error.is_instance_of::(py) || error.is_instance_of::(py) @@ -48,21 +42,13 @@ fn type_name(kind: &Bound<'_, PyAny>) -> PyResult { )) } -/// A Python `str` as JSON text; one holding a lone surrogate as UTF-16 code units. -pub(crate) fn text(value: &Bound<'_, PyString>) -> PyResult { - match value.to_str() { - Ok(text) => Ok(JsonValue::String(text.to_owned())), - Err(_) => utf16_units(value).map(JsonValue::Utf16), - } -} - /// `[tag, value]`. fn tagged(tag: String, value: JsonValue) -> JsonValue { JsonValue::Array(vec![JsonValue::String(tag), value]) } /// The Python types `_canonical` distinguishes. -struct CanonicalTypes<'py> { +pub(crate) struct CanonicalTypes<'py> { enumeration: Bound<'py, PyAny>, decimal: Bound<'py, PyAny>, dates: Bound<'py, PyTuple>, @@ -75,7 +61,7 @@ struct CanonicalTypes<'py> { } impl<'py> CanonicalTypes<'py> { - fn load(py: Python<'py>) -> PyResult { + pub(crate) fn load(py: Python<'py>) -> PyResult { Ok(Self { enumeration: imported(py, "enum", "Enum")?, decimal: imported(py, "decimal", "Decimal")?, @@ -96,7 +82,7 @@ impl<'py> CanonicalTypes<'py> { } /// `_canonical(value)`, on a stack that grows so nesting up to `MAX_DEPTH` cannot overflow it. - fn canonical(&self, value: &Bound<'py, PyAny>, depth: usize) -> PyResult { + pub(crate) fn canonical(&self, value: &Bound<'py, PyAny>, depth: usize) -> PyResult { stacker::maybe_grow(CANONICAL_RED_ZONE, CANONICAL_STACK_GROWTH, || { self.canonical_level(value, depth) }) @@ -218,85 +204,8 @@ impl<'py> CanonicalTypes<'py> { } } -/// Name a macro by its name, file, identity and exact source, or None when it has no key. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn macro_store_token<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "macro_store_token", - options, - (&["loaded", "identity"], &[]), - )?; - let loaded: Bound<'py, PyAny> = arguments.value("loaded")?; - let keyed: PyResult> = (|| { - let types: CanonicalTypes<'py> = CanonicalTypes::load(py)?; - let name: JsonValue = text(native_field(&loaded, "name")?.downcast::()?)?; - let path: JsonValue = text( - native_field(&loaded, "relative_path")? - .call_method0("as_posix")? - .downcast::()?, - )?; - let identity: JsonValue = types.canonical(&arguments.value("identity")?, 0)?; - let raw_source: String = native_field(&loaded, "raw_source")?.extract()?; - macro_key((name, path), identity, &raw_source).map_err(emit_error) - })(); - match keyed { - Err(error) if unstable(py, &error) => Ok(None), - other => other, - } -} - -/// Digest the whole context a macro may read, or None when a value has no stable text. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn context_store_token<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "context_store_token", - options, - (&["macro_context", "declarations"], &[]), - )?; - let keyed: PyResult> = (|| { - let types: CanonicalTypes<'py> = CanonicalTypes::load(py)?; - let context: JsonValue = canonical_context(&types, &arguments.value("macro_context")?)?; - let declarations: Bound<'py, PyAny> = arguments.value("declarations")?; - let declared: JsonValue = if declarations.is_none() { - JsonValue::Null - } else { - let sorted = imported(py, "builtins", "sorted")?; - let mut parts: Vec = vec![ - types.canonical(&native_field(&declarations, "constants")?, 0)?, - types.canonical(&native_field(&declarations, "enums")?, 0)?, - ]; - for name in ["inaccessible_constants", "inaccessible_enums"] { - let mut items: Vec = Vec::new(); - for item in sorted - .call1((native_field(&declarations, name)?,))? - .try_iter()? - { - items.push(plain_json(&item?)?); - } - parts.push(JsonValue::Array(items)); - } - JsonValue::Array(parts) - }; - context_key(&JsonValue::Array(vec![context, declared])).map_err(emit_error) - })(); - match keyed { - Err(error) if unstable(py, &error) => Ok(None), - other => other, - } -} - /// `_canonical_context`: the context's type and each field, the value renderer by its type. -fn canonical_context<'py>( +pub(crate) fn canonical_context<'py>( types: &CanonicalTypes<'py>, macro_context: &Bound<'py, PyAny>, ) -> PyResult { @@ -322,7 +231,7 @@ fn canonical_context<'py>( } /// An inaccessible declaration name, which is always a string, as JSON. -fn plain_json(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn plain_json(value: &Bound<'_, PyAny>) -> PyResult { match value.downcast::() { Ok(string) => text(string), Err(_) => Err(pyo3::exceptions::PyTypeError::new_err(format!( @@ -331,8 +240,3 @@ fn plain_json(value: &Bound<'_, PyAny>) -> PyResult { ))), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(macro_store_token, module)?)?; - module.add_function(wrap_pyfunction!(context_store_token, module)?) -} diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/context_store_token.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/context_store_token.rs index be5b01f271..6224727df0 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/main/context_store_token.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/context_store_token.rs @@ -1,16 +1,60 @@ //! Digest the whole context a macro may read, or None when a value has no stable text. -use crate::macro_bridge::_helpers::store_keys; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::macro_bridge::_helpers::store_keys::CanonicalTypes; +use crate::macro_bridge::_helpers::store_keys::canonical_context; +use crate::macro_bridge::_helpers::store_keys::plain_json; +use crate::macro_bridge::_helpers::store_keys::unstable; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_cache::macro_store_keys::main::context_store_token::context_store_token as context_key; +use sqlbuild_core::json::models::JsonValue; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::frontier::main::emit_error::emit_error; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Digest the whole context a macro may read, or None when a value has no stable text. -#[inline] +#[pyfunction] +#[pyo3(signature = (**options))] pub fn context_store_token<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - store_keys::context_store_token(py, options) + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "context_store_token", + options, + (&["macro_context", "declarations"], &[]), + )?; + let keyed: PyResult> = (|| { + let types: CanonicalTypes<'py> = CanonicalTypes::load(py)?; + let context: JsonValue = canonical_context(&types, &arguments.value("macro_context")?)?; + let declarations: Bound<'py, PyAny> = arguments.value("declarations")?; + let declared: JsonValue = if declarations.is_none() { + JsonValue::Null + } else { + let sorted = imported(py, "builtins", "sorted")?; + let mut parts: Vec = vec![ + types.canonical(&native_field(&declarations, "constants")?, 0)?, + types.canonical(&native_field(&declarations, "enums")?, 0)?, + ]; + for name in ["inaccessible_constants", "inaccessible_enums"] { + let mut items: Vec = Vec::new(); + for item in sorted + .call1((native_field(&declarations, name)?,))? + .try_iter()? + { + items.push(plain_json(&item?)?); + } + parts.push(JsonValue::Array(items)); + } + JsonValue::Array(parts) + }; + context_key(&JsonValue::Array(vec![context, declared])).map_err(emit_error) + })(); + match keyed { + Err(error) if unstable(py, &error) => Ok(None), + other => other, + } } diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/keep_delta_claim.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/keep_delta_claim.rs index f2e88dec30..ffb7bc548a 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/main/keep_delta_claim.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/keep_delta_claim.rs @@ -1,9 +1,9 @@ //! Keep the delta compile's claim: it finished, so no full compile of this invocation follows. -use crate::macro_bridge::_helpers::first_compile; +use crate::macro_bridge::_helpers::first_compile::FIRST_COMPILE; +use crate::macro_bridge::_helpers::first_compile::locked; /// Keep the delta compile's claim: it finished, so no full compile of this invocation follows. -#[inline] pub fn keep_delta_claim() { - first_compile::keep_delta_claim() + locked(&FIRST_COMPILE).lent_to_delta = false; } diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/macro_store_token.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/macro_store_token.rs index 1219b76282..211f76335b 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/main/macro_store_token.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/macro_store_token.rs @@ -1,16 +1,45 @@ //! Name a macro by its name, file, identity and exact source, or None when it has no key. -use crate::macro_bridge::_helpers::store_keys; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; +use crate::macro_bridge::_helpers::store_keys::CanonicalTypes; +use crate::macro_bridge::_helpers::store_keys::unstable; +use crate::macro_bridge::main::text::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyString}; +use sqlbuild_cache::macro_store_keys::main::macro_store_token::macro_store_token as macro_key; +use sqlbuild_core::json::models::JsonValue; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::frontier::main::emit_error::emit_error; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Name a macro by its name, file, identity and exact source, or None when it has no key. -#[inline] +#[pyfunction] +#[pyo3(signature = (**options))] pub fn macro_store_token<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - store_keys::macro_store_token(py, options) + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "macro_store_token", + options, + (&["loaded", "identity"], &[]), + )?; + let loaded: Bound<'py, PyAny> = arguments.value("loaded")?; + let keyed: PyResult> = (|| { + let types: CanonicalTypes<'py> = CanonicalTypes::load(py)?; + let name: JsonValue = text(native_field(&loaded, "name")?.downcast::()?)?; + let path: JsonValue = text( + native_field(&loaded, "relative_path")? + .call_method0("as_posix")? + .downcast::()?, + )?; + let identity: JsonValue = types.canonical(&arguments.value("identity")?, 0)?; + let raw_source: String = native_field(&loaded, "raw_source")?.extract()?; + macro_key((name, path), identity, &raw_source).map_err(emit_error) + })(); + match keyed { + Err(error) if unstable(py, &error) => Ok(None), + other => other, + } } diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/register_first_compile.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/register_first_compile.rs index d255b60486..5f490e2e9e 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/main/register_first_compile.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/register_first_compile.rs @@ -1,12 +1,10 @@ //! Register the first compile functions and classes. -use crate::macro_bridge::_helpers::first_compile; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::macro_bridge::_helpers::first_compile::claim_first_compile; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the first compile functions and classes. -#[inline] pub fn register_first_compile(module: &Bound<'_, PyModule>) -> PyResult<()> { - first_compile::register(module) + module.add_function(wrap_pyfunction!(claim_first_compile, module)?) } diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs index 5c4403757a..174cb2e3ee 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs @@ -1,12 +1,14 @@ //! Register the module digests functions and classes. -use crate::macro_bridge::_helpers::module_digests; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::macro_bridge::_helpers::module_digests::digest_module_files; +use crate::macro_bridge::_helpers::module_digests::module_digests_metadata; +use crate::macro_bridge::_helpers::module_digests::unchanged_module_digests; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the module digests functions and classes. -#[inline] pub fn register_module_digests(module: &Bound<'_, PyModule>) -> PyResult<()> { - module_digests::register(module) + module.add_function(wrap_pyfunction!(digest_module_files, module)?)?; + module.add_function(wrap_pyfunction!(module_digests_metadata, module)?)?; + module.add_function(wrap_pyfunction!(unchanged_module_digests, module)?) } diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/register_store_keys.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/register_store_keys.rs index 8ec80a3394..216dd80429 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/main/register_store_keys.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/register_store_keys.rs @@ -1,12 +1,12 @@ //! Register the store keys functions and classes. -use crate::macro_bridge::_helpers::store_keys; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::macro_bridge::main::context_store_token::context_store_token; +use crate::macro_bridge::main::macro_store_token::macro_store_token; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the store keys functions and classes. -#[inline] pub fn register_store_keys(module: &Bound<'_, PyModule>) -> PyResult<()> { - store_keys::register(module) + module.add_function(wrap_pyfunction!(macro_store_token, module)?)?; + module.add_function(wrap_pyfunction!(context_store_token, module)?) } diff --git a/crates/sqlbuild-python-front/src/macro_bridge/main/text.rs b/crates/sqlbuild-python-front/src/macro_bridge/main/text.rs index 4a1a649ee5..82ac2875ce 100644 --- a/crates/sqlbuild-python-front/src/macro_bridge/main/text.rs +++ b/crates/sqlbuild-python-front/src/macro_bridge/main/text.rs @@ -1,13 +1,14 @@ //! A Python `str` as JSON text; one holding a lone surrogate as UTF-16 code units. -use crate::macro_bridge::_helpers::store_keys; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyString; +use pyo3::prelude::{Bound, PyResult}; +use pyo3::types::{PyString, PyStringMethods}; use sqlbuild_core::json::models::JsonValue; +use sqlbuild_python_base::boundary::main::utf16_units::utf16_units; /// A Python `str` as JSON text; one holding a lone surrogate as UTF-16 code units. -#[inline] pub fn text(value: &Bound<'_, PyString>) -> PyResult { - store_keys::text(value) + match value.to_str() { + Ok(text) => Ok(JsonValue::String(text.to_owned())), + Err(_) => utf16_units(value).map(JsonValue::Utf16), + } } diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs index 94efd6949a..6d4d27fd1b 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs @@ -1,19 +1,17 @@ //! The Python classes a model's `CompileModelConfig` is built from, and building one from a native build. +use crate::model_config::main::config::config_fields::config_fields; +use crate::model_config::main::config::view::view; use crate::model_config::models::ConfigClasses; -use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; -use sqlbuild_model_config::model_validation::models::{RetentionOverride, TableTypeOverride}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; -use crate::model_config::_helpers::config::typed_config::retain_values; use crate::model_config::models::BuiltConfig; -use crate::model_config::types::BuiltConfigValues; const MATERIALIZED_KEY: &str = "materialized"; -const VALUES_FIELD: &str = "values"; -const PERMANENT_TABLE_TYPE: &str = "permanent"; -const TRANSIENT_TABLE_TYPE: &str = "transient"; +pub(crate) const VALUES_FIELD: &str = "values"; +pub(crate) const PERMANENT_TABLE_TYPE: &str = "permanent"; +pub(crate) const TRANSIENT_TABLE_TYPE: &str = "transient"; /// The model's `CompileModelConfig`, once its template reads are recorded. pub(crate) fn built_config<'py>( @@ -25,61 +23,8 @@ pub(crate) fn built_config<'py>( view(classes, &built.values, &fields, built.nodes.as_ref()) } -/// Every `CompileModelConfig` field but `values`, recording the build's template reads first. -pub(crate) fn config_fields<'py>( - classes: &ConfigClasses<'py>, - built: &BuiltConfig<'py>, - matched: Option<&str>, -) -> PyResult> { - let py: Python<'py> = built.values.py(); - if !built.reads.is_empty() { - classes - .record_template_reads - .call1((PyTuple::new(py, &built.reads)?,))?; - } - let none: Bound<'py, PyAny> = py.None().into_bound(py); - let (inherited_retention, inherited_table_type): (Bound<'py, PyAny>, Bound<'py, PyAny>) = - classes - .inherited_storage - .as_any() - .get_item(storage_key(classes, &built.values)?)? - .extract()?; - let retention: Bound<'py, PyAny> = match built.retention { - RetentionOverride::Inherit => inherited_retention, - RetentionOverride::Unmanaged => resolved_retention(classes, none.clone(), true)?, - RetentionOverride::Days(days) => { - resolved_retention(classes, days.into_pyobject(py)?.into_any(), false)? - } - }; - let table_type: Bound<'py, PyAny> = match built.table_type { - TableTypeOverride::Inherit => inherited_table_type, - TableTypeOverride::Permanent => resolved_table_type(classes, PERMANENT_TABLE_TYPE)?, - TableTypeOverride::Transient => resolved_table_type(classes, TRANSIENT_TABLE_TYPE)?, - }; - let logical_schema: Bound<'py, PyAny> = built.logical_schema.clone().unwrap_or(none.clone()); - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item("model_header_keys", PyTuple::new(py, &built.header_keys)?)?; - fields.set_item("matched_path_default", matched)?; - fields.set_item("logical_schema", &logical_schema)?; - fields.set_item( - "layer_schema", - if built.layer_schema_configured { - logical_schema - } else { - none.clone() - }, - )?; - fields.set_item( - "logical_database", - built.logical_database.clone().unwrap_or(none), - )?; - fields.set_item("time_travel_retention", retention)?; - fields.set_item("table_type", table_type)?; - Ok(fields) -} - /// The materialization whose inherited storage policy applies, or `None` when not table-backed. -fn storage_key<'py>( +pub(crate) fn storage_key<'py>( classes: &ConfigClasses<'py>, values: &Bound<'py, PyDict>, ) -> PyResult> { @@ -116,17 +61,3 @@ pub(crate) fn resolved_table_type<'py>( fields.set_item("declared", true)?; classes.resolved_table_type.call((), Some(&fields)) } - -/// A `CompileModelConfig` of `values` and the other `fields`. -pub(crate) fn view<'py>( - classes: &ConfigClasses<'py>, - values: &Bound<'py, PyDict>, - fields: &Bound<'py, PyDict>, - nodes: Option<&BuiltConfigValues>, -) -> PyResult> { - let arguments: Bound<'py, PyDict> = fields.copy()?; - arguments.set_item(VALUES_FIELD, values)?; - let config = classes.compile_model_config.call((), Some(&arguments))?; - retain_values(&config, nodes)?; - Ok(config) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_errors.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_errors.rs index 1ff777cbd3..77a9eb366d 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_errors.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_errors.rs @@ -1,9 +1,8 @@ //! Native config errors handed to Python, which raises them as the Python stages would. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pymethods; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{Py, PyErr, pymethods}; -use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; use crate::model_config::_helpers::config::config_values::config_value_type_error; use crate::model_config::errors::NativeConfigError; @@ -12,33 +11,10 @@ use sqlbuild_python_base::records::types::ErrorId; const EXPECTED_STRING: &str = "a string"; -const COMPILE_INPUT_CLASS: &str = "compile_input"; -const CONFIG_VALUE_TYPE_CLASS: &str = "config_value_type"; -const RESOURCE_IDENTITY_CLASS: &str = "resource_identity"; -const DISCOVERY_CONFLICT_CLASS: &str = "discovery_conflict"; - -/// Wrap `error` for Python. -pub(crate) fn native_config_error( - py: Python<'_>, - error: ConfigError, -) -> PyResult> { - let class_name = match error.class { - ErrorClass::CompileInput => COMPILE_INPUT_CLASS, - ErrorClass::ConfigValueType => CONFIG_VALUE_TYPE_CLASS, - ErrorClass::ResourceIdentity => RESOURCE_IDENTITY_CLASS, - ErrorClass::DiscoveryConflict => DISCOVERY_CONFLICT_CLASS, - }; - Py::new( - py, - NativeConfigError { - class_name, - message: error.message, - code: error.code, - help: error.help, - key: error.key, - }, - ) -} +pub(crate) const COMPILE_INPUT_CLASS: &str = "compile_input"; +pub(crate) const CONFIG_VALUE_TYPE_CLASS: &str = "config_value_type"; +pub(crate) const RESOURCE_IDENTITY_CLASS: &str = "resource_identity"; +pub(crate) const DISCOVERY_CONFLICT_CLASS: &str = "discovery_conflict"; /// The Python exception a native config error raises as, given the values it names. pub(crate) fn config_exception<'py>( @@ -49,17 +25,6 @@ pub(crate) fn config_exception<'py>( error.get().exception(error.py(), values) } -/// `config_exception` as the raised error, or the failure building it. -pub(crate) fn raised_config_exception( - error: &Bound<'_, PyAny>, - values: Option<&Bound<'_, PyAny>>, -) -> PyErr { - match config_exception(error, values) { - Ok(instance) => PyErr::from_value(instance), - Err(failure) => failure, - } -} - #[pymethods] impl NativeConfigError { /// The exception this error raises as: its class, message, code and help. @@ -91,8 +56,3 @@ impl NativeConfigError { error_class(py, class)?.call((&self.message,), Some(&keywords)) } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_nodes.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_nodes.rs index 33b0181197..b3f1843564 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_nodes.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_nodes.rs @@ -1,10 +1,11 @@ //! Python config values converted once into `ConfigNode`s, and projected back into Python objects. +use crate::model_config::main::config::config_node::config_node; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::models::HostHandle; use crate::model_config::types::PyConfigNode; use std::sync::Arc; -use pyo3::IntoPyObject; use pyo3::exceptions::{PyOverflowError, PyUnicodeEncodeError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{ @@ -40,7 +41,10 @@ fn utf8_text(text: &Bound<'_, PyString>) -> PyResult { } /// An opaque value with the cheap facts `PyNode` reads of it, and its converted items. -fn host(value: &Bound<'_, PyAny>, children: HostChildren) -> PyResult { +pub(crate) fn host( + value: &Bound<'_, PyAny>, + children: HostChildren, +) -> PyResult { let node: PyNode<'_> = PyNode(value.clone()); let strict_text: bool = match value.downcast::() { Ok(text) => utf8_text(text)?, @@ -109,7 +113,7 @@ fn container( }) } -fn convert(value: &Bound<'_, PyAny>, open: &[usize]) -> PyResult> { +pub(crate) fn convert(value: &Bound<'_, PyAny>, open: &[usize]) -> PyResult> { if value.is_none() { return Ok(Some(ConfigNode::Null)); } @@ -147,15 +151,6 @@ fn convert(value: &Bound<'_, PyAny>, open: &[usize]) -> PyResult) -> PyResult { - if is_exact_container(value) { - host(value, HostChildren::None) - } else { - config_node(value) - } -} - /// Whether `value` is an exact `list`, `tuple` or `dict`. pub(crate) fn is_exact_container(value: &Bound<'_, PyAny>) -> bool { value.is_exact_instance_of::() @@ -163,14 +158,6 @@ pub(crate) fn is_exact_container(value: &Bound<'_, PyAny>) -> bool { || value.is_exact_instance_of::() } -/// One Python config value as a `ConfigNode`; a value that holds itself stays opaque whole. -pub(crate) fn config_node(value: &Bound<'_, PyAny>) -> PyResult { - match convert(value, &[])? { - Some(node) => Ok(node), - None => host(value, HostChildren::None), - } -} - /// A Python mapping's entries as config entries. pub(crate) fn config_entries(mapping: &Bound<'_, PyDict>) -> PyResult { mapping @@ -179,38 +166,6 @@ pub(crate) fn config_entries(mapping: &Bound<'_, PyDict>) -> PyResult( - py: Python<'py>, - node: &PyConfigNode, -) -> PyResult> { - Ok(match node { - ConfigNode::Null => py.None().into_bound(py), - ConfigNode::Bool(flag) => PyBool::new(py, *flag).to_owned().into_any(), - ConfigNode::Int(number) => number.into_pyobject(py)?.into_any(), - ConfigNode::Float(number) => PyFloat::new(py, *number).into_any(), - ConfigNode::Str(text) => PyString::new(py, text).into_any(), - ConfigNode::List(items) => PyList::new( - py, - items - .iter() - .map(|item| config_object(py, item)) - .collect::>>()?, - )? - .into_any(), - ConfigNode::Tuple(items) => PyTuple::new( - py, - items - .iter() - .map(|item| config_object(py, item)) - .collect::>>()?, - )? - .into_any(), - ConfigNode::Map(entries) => config_dict(py, entries)?.into_any(), - ConfigNode::Host(host) => host.handle.0.bind(py).clone(), - }) -} - /// A new Python `dict` of config entries. pub(crate) fn config_dict<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs index 759e89e53f..f22d92a421 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs @@ -1,26 +1,21 @@ //! `${...}` templates in authored Python config values and effective vars, expanded natively. +use crate::model_config::main::config::expanded::expanded; +use crate::model_config::main::config::native_config_error::native_config_error; +use crate::model_config::main::config::template_error::template_error; use crate::model_config::models::PythonHost; use crate::model_config::models::Stop; use crate::model_config::models::TemplateSources; use std::cell::RefCell; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyResult, Python}; -use pyo3::types::{ - PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, - PyTupleMethods, -}; -use sqlbuild_model_config::errors::ConfigError; -use sqlbuild_model_config::templates::errors::TemplateError; -use sqlbuild_model_config::templates::main::expand_template_string::expand_template_string; -use sqlbuild_model_config::templates::main::template_error_message::template_error_message; -use sqlbuild_model_config::templates::models::{StringExpansion, TemplateOptions}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTuple}; +use sqlbuild_model_config::templates::models::TemplateOptions; use sqlbuild_model_config::templates::types::TemplateHost; -use crate::model_config::_helpers::config::config_errors::native_config_error; use crate::model_config::errors::NativeConfigError; -const TEMPLATE_OPEN_TOKEN: &str = "${"; +pub(crate) const TEMPLATE_OPEN_TOKEN: &str = "${"; pub(crate) const ENVIRONMENT_READ: &str = "env"; pub(crate) const CONTEXT_READ: &str = "ctx"; pub(crate) const EFFECTIVE_VARS_LABEL: &str = "effective vars"; @@ -38,11 +33,6 @@ pub(crate) struct EffectiveVars<'py> { pub(crate) resolving: RefCell>, } -/// The `CompileInputError` Python raises for a template error in the context `label`. -pub(crate) fn template_error(error: &TemplateError, label: &str) -> ConfigError { - ConfigError::compile(template_error_message(error, label)) -} - /// The expanded value, or the native error expansion stopped at, and the reads in order. pub(crate) type TemplateOutcome<'py> = ( Result, Py>, @@ -160,52 +150,3 @@ fn outcome<'py>( }; Ok((value, host.into_reads())) } - -/// Expand every template in `value` and the containers inside it. -pub(crate) fn expanded<'py>( - host: &PythonHost<'py>, - value: &Bound<'py, PyAny>, - options: TemplateOptions, -) -> Result, Stop> { - let py = host.py; - if let Ok(text) = value.downcast::() { - let text = text.to_str().map_err(Stop::Python)?; - if !text.contains(TEMPLATE_OPEN_TOKEN) { - return Ok(value.clone()); - } - return match expand_template_string(host, text, options) { - Ok(StringExpansion::Unchanged) => Ok(PyString::new(py, text).into_any()), - Ok(StringExpansion::Value(result)) => Ok(result), - Ok(StringExpansion::Text(result)) => Ok(PyString::new(py, &result).into_any()), - Err(failure) => Err(host.stop(failure)), - }; - } - if let Ok(mapping) = value.downcast::() { - let result = PyDict::new(py); - for (key, item) in mapping.iter() { - result - .set_item(key, expanded(host, &item, options)?) - .map_err(Stop::Python)?; - } - return Ok(result.into_any()); - } - if let Ok(items) = value.downcast::() { - let result = items - .iter() - .map(|item| expanded(host, &item, options)) - .collect::, _>>()?; - return PyList::new(py, result) - .map(Bound::into_any) - .map_err(Stop::Python); - } - if let Ok(items) = value.downcast::() { - let result = items - .iter() - .map(|item| expanded(host, &item, options)) - .collect::, _>>()?; - return PyTuple::new(py, result) - .map(Bound::into_any) - .map_err(Stop::Python); - } - Ok(value.clone()) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs index 8cf5b4efed..bdeb7dc758 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs @@ -1,16 +1,15 @@ //! Typed values read from raw authored configuration mappings, rejecting the wrong types. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use crate::model_config::main::config::config_node::config_node; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::{PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::config_values::main::checked_cursor_bound::checked_cursor_bound; use sqlbuild_model_config::config_values::main::checked_flag::checked_flag; use sqlbuild_model_config::config_values::main::checked_integer::checked_integer; -use sqlbuild_model_config::config_values::main::checked_text::checked_text; use sqlbuild_model_config::config_values::main::checked_texts::checked_texts; use sqlbuild_model_config::config_values::models::CursorBound; -use crate::model_config::_helpers::config::config_nodes::config_node; use crate::model_config::_helpers::config::typed_config::type_error; use crate::model_config::types::PyConfigNode; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -47,7 +46,7 @@ pub(crate) fn config_value_type_error<'py>( } /// `values.get(key)` as a config node. -fn authored_node<'py>( +pub(crate) fn authored_node<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult<(PyConfigNode, Bound<'py, PyAny>)> { @@ -55,19 +54,6 @@ fn authored_node<'py>( Ok((config_node(&value)?, value)) } -/// An authored string, preserving a present empty string. -#[pyfunction(name = "get_config_str")] -#[pyo3(signature = (*, values, key))] -pub(crate) fn config_str<'py>( - values: &Bound<'py, PyAny>, - key: &str, -) -> PyResult>> { - let (node, value) = authored_node(values, key)?; - let checked = - checked_text(key, Some(&node)).map_err(|error| type_error(values.py(), &error))?; - Ok(checked.map(|_| value)) -} - /// An authored boolean, or `None` when the key is absent or `None`. #[pyfunction(name = "get_config_bool")] #[pyo3(signature = (*, values, key))] @@ -127,12 +113,3 @@ pub(crate) fn config_cursor_bound<'py>( Some(CursorBound::Stringified(_)) => Some(value.str()?.into_any()), }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(config_str, module)?)?; - module.add_function(wrap_pyfunction!(config_bool, module)?)?; - module.add_function(wrap_pyfunction!(config_string_tuple, module)?)?; - module.add_function(wrap_pyfunction!(config_int, module)?)?; - module.add_function(wrap_pyfunction!(config_cursor_bound, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs index 5a8c4bb375..4e4eae0527 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs @@ -1,55 +1,33 @@ //! Template expansion for compile inputs and effective vars, with the reads it makes recorded. -use std::time::{SystemTime, UNIX_EPOCH}; - -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::scenario_command::main::format_captured_at::format_captured_at; +use crate::model_config::main::config::expanded_effective_vars::expanded_effective_vars; +use crate::model_config::main::config::expanded_template_data::expanded_template_data; +use crate::model_config::main::config::record_reads::record_reads; +use crate::model_config::main::config::run_id::run_id; +use crate::model_config::main::model_nodes::unicode_environment::unicode_environment; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; use sqlbuild_model_config::templates::models::TemplateOptions; use crate::model_config::_helpers::config::config_errors::config_exception; use crate::model_config::_helpers::config::config_templates::TemplateOutcome; -use crate::model_config::_helpers::config::config_templates::expand_config_templates; -use crate::model_config::_helpers::config::config_templates::lone_surrogate_site; -use crate::model_config::_helpers::config::config_templates::resolve_effective_vars; -use crate::model_config::_helpers::model_nodes::unicode_environment::unicode_environment; -use crate::model_config::models::TemplateSources; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; -use sqlbuild_python_base::compile_context::main::input_reads::context_read::context_read; -use sqlbuild_python_base::compile_context::main::input_reads::environment_read::environment_read; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::types::ErrorId; - -const UUID_MODULE: &str = "uuid"; -const ENVIRONMENT_READ: &str = "env"; -const RUN_ID_KEY: &str = "run.id"; -const RUN_TARGET_KEY: &str = "run.target"; -const MODEL_CONFIG_LABEL: &str = "model config"; -const RUN_ID_SUFFIX_LENGTH: usize = 12; -const TIMESTAMP_SEPARATORS: [char; 2] = ['-', ':']; -/// Record native template environment and context reads for project reuse, in order. -pub(crate) fn record_reads<'a>( - py: Python<'_>, - reads: impl IntoIterator, -) -> PyResult<()> { - for (kind, name) in reads { - if kind == ENVIRONMENT_READ { - environment_read(py, name)?; - } else { - context_read(py, name)?; - } - } - Ok(()) -} +pub(crate) const UUID_MODULE: &str = "uuid"; +pub(crate) const ENVIRONMENT_READ: &str = "env"; +pub(crate) const RUN_ID_KEY: &str = "run.id"; +pub(crate) const RUN_TARGET_KEY: &str = "run.target"; +pub(crate) const MODEL_CONFIG_LABEL: &str = "model config"; +pub(crate) const RUN_ID_SUFFIX_LENGTH: usize = 12; +pub(crate) const TIMESTAMP_SEPARATORS: [char; 2] = ['-', ':']; /// Record `outcome`'s reads, then return its value or raise its error as Python's stage does. -fn applied<'py>(py: Python<'py>, outcome: TemplateOutcome<'py>) -> PyResult> { +pub(crate) fn applied<'py>( + py: Python<'py>, + outcome: TemplateOutcome<'py>, +) -> PyResult> { let (value, reads) = outcome; record_reads(py, reads.iter().map(|(kind, name)| (*kind, name.as_str())))?; match value { @@ -62,106 +40,16 @@ fn applied<'py>(py: Python<'py>, outcome: TemplateOutcome<'py>) -> PyResult) -> PyResult> { +pub(crate) fn environment(py: Python<'_>) -> PyResult> { unicode_environment(py) } -/// `value` with every `${...}` template expanded, recording the reads it made. -pub(crate) fn expanded_template_data<'py>( - value: &Bound<'py, PyAny>, - (variables, context_values): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), - (options, label): (TemplateOptions, &str), -) -> PyResult> { - let py: Python<'py> = value.py(); - let sources = TemplateSources(variables.clone(), environment(py)?, context_values.clone()); - applied( - py, - expand_config_templates(py, value, sources, (options, label))?, - ) -} - -/// Merged effective vars with recursive `${name}` expansion. -pub(crate) fn expanded_effective_vars<'py>( - raw_values: &Bound<'py, PyDict>, -) -> PyResult> { - let py: Python<'py> = raw_values.py(); - if let Some((name, path, byte_offset)) = lone_surrogate_site(raw_values)? { - let error: Bound<'py, PyAny> = error_class(py, ErrorId::CompileInput)?.call( - (format!( - "Variable '{name}' holds a lone surrogate in {path} at UTF-8 byte \ - {byte_offset}, which is not valid Unicode text" - ),), - Some(&keywords( - py, - &[( - "help", - PyString::new( - py, - &format!( - "Set '{name}' to valid Unicode text in --vars or the [vars] \ - table; the value is not shown because it may be a secret" - ), - ) - .into_any(), - )], - )?), - )?; - return Err(PyErr::from_value(error)); - } - applied( - py, - resolve_effective_vars(py, raw_values, environment(py)?)?, - ) -} - -/// Model header `values` with `${name}`, `${ENV:...}` and early `run.*` templates resolved. -pub(crate) fn early_model_templates<'py>( - values: &Bound<'py, PyAny>, - effective_vars: &Bound<'py, PyAny>, - (target_name, run_id): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - let py: Python<'py> = values.py(); - let context_values: Bound<'py, PyDict> = PyDict::new(py); - context_values.set_item(RUN_ID_KEY, run_id)?; - context_values.set_item(RUN_TARGET_KEY, target_name)?; - expanded_template_data( - values, - (effective_vars.downcast()?, &context_values), - ( - TemplateOptions { - allow_context: true, - preserve_context_tokens: false, - preserve_unknown_context: true, - }, - MODEL_CONFIG_LABEL, - ), - ) -} - -/// The selected run id, or a new `YYYYMMDDTHHMMSSZ_<12 hex>` compile invocation id. -pub(crate) fn run_id<'py>(selected_run_id: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = selected_run_id.py(); - if !selected_run_id.is_none() { - return Ok(selected_run_id.clone()); - } - let epoch_seconds: i64 = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_or(0, |elapsed| { - i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX) - }); - let timestamp: String = format_captured_at(epoch_seconds).replace(TIMESTAMP_SEPARATORS, ""); - let suffix: String = imported(py, UUID_MODULE, "uuid4")? - .call0()? - .getattr("hex")? - .extract()?; - let suffix: &str = suffix.get(..RUN_ID_SUFFIX_LENGTH).unwrap_or(&suffix); - Ok(PyString::new(py, &format!("{timestamp}_{suffix}")).into_any()) -} - /// Recursively expand template strings inside supported Python container values. #[pyfunction] #[pyo3(signature = (**request))] -fn expand_template_data<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn expand_template_data<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request: Request<'_, 'py> = Request::new("expand_template_data", request)?; let flag = |name: &str| -> PyResult { request.get(name)?.is_truthy() }; expanded_template_data( @@ -183,7 +71,9 @@ fn expand_template_data<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult(raw_values: &Bound<'py, PyDict>) -> PyResult> { +pub(crate) fn expand_effective_vars<'py>( + raw_values: &Bound<'py, PyDict>, +) -> PyResult> { expanded_effective_vars(raw_values) } @@ -201,21 +91,8 @@ pub(crate) fn record_template_reads(py: Python<'_>, reads: Vec<(String, String)> /// Resolve a stable compile invocation id. #[pyfunction] #[pyo3(signature = (*, selected_run_id))] -fn resolve_run_id<'py>(selected_run_id: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn resolve_run_id<'py>( + selected_run_id: &Bound<'py, PyAny>, +) -> PyResult> { run_id(selected_run_id) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(expand_template_data, module)?)?; - module.add_function(wrap_pyfunction!(expand_effective_vars, module)?)?; - module.add_function(wrap_pyfunction!(record_template_reads, module)?)?; - module.add_function(wrap_pyfunction!(resolve_run_id, module)?)?; - Ok(()) -} - -/// `record_template_reads` as a Python function. -pub(crate) fn record_template_reads_function( - py: pyo3::Python<'_>, -) -> pyo3::PyResult> { - pyo3::wrap_pyfunction!(record_template_reads, py) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/config/typed_config.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/config/typed_config.rs index 5d628ab405..3b27a96718 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/config/typed_config.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/config/typed_config.rs @@ -1,6 +1,7 @@ //! Built model config trees retained natively, with Python-authored values converted on first read. -use crate::model_config::models::TypedConfigValues; +use crate::model_config::main::config::config_node::config_node; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::types::BuiltConfigValues; use crate::model_config::types::ConfigValues; use std::collections::HashMap; @@ -8,35 +9,23 @@ use std::sync::{Arc, Mutex}; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyString, PyTuple}; +use pyo3::types::PyString; use sqlbuild_model_config::config_values::errors::ConfigTypeError; -use sqlbuild_model_config::config_values::main::checked_cursor_bound::checked_cursor_bound; -use sqlbuild_model_config::config_values::main::checked_flag::checked_flag; -use sqlbuild_model_config::config_values::main::checked_integer::checked_integer; -use sqlbuild_model_config::config_values::main::checked_text::checked_text; -use sqlbuild_model_config::config_values::main::checked_texts::checked_texts; use sqlbuild_model_config::config_values::models::{ConfigNode, CursorBound, HostChildren}; -use crate::model_config::_helpers::config::config_nodes::PyConfigEntries; -use crate::model_config::_helpers::config::config_nodes::config_node; -use crate::model_config::_helpers::config::config_nodes::config_object; -use crate::model_config::_helpers::config::config_nodes::is_exact_container; +use crate::model_config::_helpers::config::config_nodes::{PyConfigEntries, is_exact_container}; use crate::model_config::_helpers::config::config_values::config_value_type_error; use crate::model_config::models::HostHandle; use crate::model_config::types::PyConfigNode; use sqlbuild_python_base::compile_driver::constants::TYPED_CONFIG_READS; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::cached_value::cached_value; -use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_base::records::main::fields::store_cached::store_cached; -use sqlbuild_python_base::records::models::NativeRecord; -const CONFIG_FIELD: &str = "config"; -const VALUES_FIELD: &str = "values"; +pub(crate) const CONFIG_FIELD: &str = "config"; +pub(crate) const VALUES_FIELD: &str = "values"; const ISOFORMAT_METHOD: &str = "isoformat"; /// The config record's cache slot holding its typed values; no Python attribute reads it. -const TYPED_VALUES_SLOT: &str = "__typed_values__"; +pub(crate) const TYPED_VALUES_SLOT: &str = "__typed_values__"; const POISONED: &str = "typed config values cache is poisoned"; /// Keep the builder's tree without interpreting custom Python mapping keys natively. @@ -51,28 +40,8 @@ pub(crate) fn built_values(entries: PyConfigEntries) -> Option, - built: Option<&BuiltConfigValues>, -) -> PyResult<()> { - if let Some(built) = built - && let Ok(record) = config.cast::() - { - let typed = Bound::new( - config.py(), - TypedConfigValues { - built: Some(Arc::clone(built)), - read: Mutex::new(Vec::new()), - }, - )?; - store_cached(record, TYPED_VALUES_SLOT, &typed.into_any().unbind())?; - } - Ok(()) -} - /// `values.get(key)` as a typed node, `None` when absent; each key is converted once. -fn node(values: &ConfigValues<'_>, key: &str) -> PyResult> { +pub(crate) fn node(values: &ConfigValues<'_>, key: &str) -> PyResult> { let (mapping, typed) = values; let py: Python<'_> = mapping.py(); let built = typed.get().built.as_ref().map(|values| values.get(key)); @@ -111,43 +80,6 @@ fn node(values: &ConfigValues<'_>, key: &str) -> PyResult> { Ok(node) } -/// A config record's values, their typed nodes kept on the record so each key converts once. -pub(crate) fn typed_values<'py>(config: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = config.py(); - let values: Bound<'py, PyAny> = native_field(config, VALUES_FIELD)?; - let fresh = || { - Bound::new( - py, - TypedConfigValues { - built: None, - read: Mutex::new(Vec::new()), - }, - ) - }; - let Ok(record) = config.cast::() else { - let typed: Bound<'py, TypedConfigValues> = fresh()?; - return Ok((values, typed)); - }; - let typed: Bound<'py, TypedConfigValues> = match cached_value(record, TYPED_VALUES_SLOT)? { - Some(typed) => typed.into_bound(py).cast_into()?, - None => { - let typed: Bound<'py, TypedConfigValues> = fresh()?; - store_cached( - record, - TYPED_VALUES_SLOT, - &typed.clone().into_any().unbind(), - )?; - typed - } - }; - Ok((values, typed)) -} - -/// A model's config values as typed nodes, read key by key. -pub(crate) fn model_values<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - typed_values(&native_field(model, CONFIG_FIELD)?) -} - /// The `ConfigValueTypeError` a typed read raises, naming the value's Python type. pub(crate) fn type_error(py: Python<'_>, error: &ConfigTypeError<'_, HostHandle>) -> PyErr { let raised = || -> PyResult { @@ -172,51 +104,6 @@ pub(crate) fn node_text(py: Python<'_>, node: &PyConfigNode) -> PyResult } } -/// `key`'s string value, or `None` when absent or `None`. -pub(crate) fn config_text( - py: Python<'_>, - values: &ConfigValues<'_>, - key: &str, -) -> PyResult> { - let node: Arc = node(values, key)?; - checked_text(key, Some(&node)) - .map_err(|error| type_error(py, &error))? - .map(|value| node_text(py, value)) - .transpose() -} - -/// `key`'s string value as its Python object, or `None` when absent or `None`. -pub(crate) fn config_text_object<'py>( - py: Python<'py>, - values: &ConfigValues<'_>, - key: &str, -) -> PyResult>> { - let node: Arc = node(values, key)?; - checked_text(key, Some(&node)) - .map_err(|error| type_error(py, &error))? - .map(|value| config_object(py, value)) - .transpose() -} - -/// Whether `key` holds a string equal to `expected`, as Python's `==` compares them. -pub(crate) fn config_text_is( - py: Python<'_>, - values: &ConfigValues<'_>, - key: &str, - expected: &str, -) -> PyResult { - let node: Arc = node(values, key)?; - let text: Option<&PyConfigNode> = - checked_text(key, Some(&node)).map_err(|error| type_error(py, &error))?; - Ok(match text { - Some(ConfigNode::Str(text)) => text == expected, - Some(ConfigNode::Host(host)) if host.facts.strict_text => { - host.facts.text.as_deref() == Some(expected) - } - _ => false, - }) -} - /// A cursor bound's text object: a string's or integer's `str()`, a date's `isoformat()`. pub(crate) fn cursor_bound_object<'py>( py: Python<'py>, @@ -231,72 +118,3 @@ pub(crate) fn cursor_bound_object<'py>( CursorBound::Stringified(value) => Ok(config_object(py, value)?.str()?.into_any()), } } - -/// `key`'s cursor bound as text, or `None` when absent or `None`. -pub(crate) fn config_cursor_bound<'py>( - py: Python<'py>, - values: &ConfigValues<'_>, - key: &str, -) -> PyResult>> { - let node: Arc = node(values, key)?; - checked_cursor_bound(key, Some(&node)) - .map_err(|error| type_error(py, &error))? - .map(|bound| cursor_bound_object(py, bound)) - .transpose() -} - -/// `values.get(key)` as its Python object, `None` when absent. -pub(crate) fn config_value<'py>( - py: Python<'py>, - values: &ConfigValues<'_>, - key: &str, -) -> PyResult> { - let value: Arc = node(values, key)?; - config_object(py, &value) -} - -/// `values.get(key)` as its typed node, `None` (null) when absent. -pub(crate) fn config_node_of(values: &ConfigValues<'_>, key: &str) -> PyResult> { - node(values, key) -} - -/// `key`'s boolean value, or `None` when absent or `None`. -pub(crate) fn config_flag( - py: Python<'_>, - values: &ConfigValues<'_>, - key: &str, -) -> PyResult> { - let node: Arc = node(values, key)?; - checked_flag(key, Some(&node)).map_err(|error| type_error(py, &error)) -} - -/// `key`'s integer value as its Python object, or `None` when absent or `None`. -pub(crate) fn config_integer_object<'py>( - py: Python<'py>, - values: &ConfigValues<'_>, - key: &str, -) -> PyResult>> { - let node: Arc = node(values, key)?; - checked_integer(key, Some(&node)) - .map_err(|error| type_error(py, &error))? - .map(|value| config_object(py, value)) - .transpose() -} - -/// `key`'s strings as a tuple, or `None` when absent or `None`. -pub(crate) fn config_text_tuple<'py>( - py: Python<'py>, - values: &ConfigValues<'_>, - key: &str, -) -> PyResult>> { - let node: Arc = node(values, key)?; - let Some(items) = checked_texts(key, Some(&node)).map_err(|error| type_error(py, &error))? - else { - return Ok(None); - }; - let objects: Vec> = items - .into_iter() - .map(|item| config_object(py, item)) - .collect::>()?; - PyTuple::new(py, objects).map(Some) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_audits.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_audits.rs index 1811576f97..1ffb1169b1 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_audits.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_audits.rs @@ -1,5 +1,7 @@ //! Retained audit contracts; Python-authored factory results remain opaque values. +use crate::model_config::main::config::config_node::config_node; +use crate::model_config::main::config::config_object::config_object; use pyo3::prelude::{PyResult, Python}; use sqlbuild_core::json::models::{JsonInteger, JsonValue}; use sqlbuild_discovery::model_files::main::config_value::config_value; @@ -8,8 +10,6 @@ use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_model_config::header_metadata::models::ParsedAudit; use sqlbuild_sqltext::compiler::models::AuthoredValue; -use crate::model_config::_helpers::config::config_nodes::config_node; -use crate::model_config::_helpers::config::config_nodes::config_object; use crate::model_config::types::HeaderValue; use crate::model_config::types::PyConfigNode; use crate::sqltext::main::authored_value_to_python::authored_value_to_python; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_columns.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_columns.rs index 86f53dccfa..29d7ed5ff6 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_columns.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_columns.rs @@ -1,20 +1,13 @@ //! Authored MODEL and SCHEMA columns parsed natively into contract columns. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use crate::model_config::main::config::native_config_error::native_config_error; +use crate::model_config::main::headers::schema_columns::schema_columns; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; -use sqlbuild_model_config::header_metadata::main::parse_schema_columns::parse_schema_columns as parse_columns; -use sqlbuild_model_config::header_metadata::models::ColumnRequest; use crate::model_config::_helpers::config::config_errors::config_exception; -use crate::model_config::_helpers::config::config_errors::native_config_error; -use crate::model_config::_helpers::headers::header_metadata::column_tuple; -use crate::model_config::_helpers::model_nodes::contract_classes::contract_classes; -use crate::model_config::models::ContractClasses; -use crate::model_config::models::PyNode; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; @@ -24,31 +17,9 @@ pub(crate) type ColumnSite<'a> = (&'a str, &'a str); /// `(require_columns, allow_migrate_from, declaration_errors)`. pub(crate) type ColumnOptions = (bool, bool, bool); -/// The `SchemaColumn` tuple of one authored columns mapping, raising Python's error for it. -pub(crate) fn schema_columns<'py>( - raw_columns: &Bound<'py, PyAny>, - site: ColumnSite<'_>, - options: ColumnOptions, - column_locations: &Bound<'py, PyDict>, -) -> PyResult> { - let py: Python<'py> = raw_columns.py(); - let (path, label) = site; - let (require_columns, allow_migrate_from, declaration_errors) = options; - let request = ColumnRequest { - path, - label, - require_columns, - allow_migrate_from, - }; - let columns = parse_columns(&PyNode(raw_columns.clone()), request) - .map_err(|error| raised(py, error, declaration_errors))?; - let classes: ContractClasses<'py> = contract_classes(py)?.extract()?; - column_tuple(py, &classes, &columns, column_locations) -} - /// Parse one MODEL or SCHEMA `columns` mapping into contract columns. #[pyfunction] -fn parse_schema_columns<'py>( +pub(crate) fn parse_schema_columns<'py>( raw_columns: Bound<'py, PyAny>, site: (String, String), options: ColumnOptions, @@ -58,7 +29,7 @@ fn parse_schema_columns<'py>( } /// A plain parse error as `DeclarationParseError` when asked; every other error as Python raises it. -fn raised(py: Python<'_>, error: ConfigError, declaration_errors: bool) -> PyErr { +pub(crate) fn raised(py: Python<'_>, error: ConfigError, declaration_errors: bool) -> PyErr { let built = (|| { if declaration_errors && error.class == ErrorClass::CompileInput { return error_class(py, ErrorId::DeclarationParse)?.call1((error.message,)); @@ -71,7 +42,3 @@ fn raised(py: Python<'_>, error: ConfigError, declaration_errors: bool) -> PyErr Err(failure) => failure, } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_schema_columns, module)?) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_families.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_families.rs deleted file mode 100644 index 144dc5045a..0000000000 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_families.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! Native dynamic-column contracts with a shared, lazy public projection. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_model_config::header_metadata::models::DynamicFamily; - -use sqlbuild_python_base::records::main::fields::field::field; - -/// A Python-authored family contract crosses into native ownership once. -pub(crate) fn family_specs(families: &Bound<'_, PyAny>) -> PyResult> { - families - .try_iter()? - .map(|family| { - let family = family?; - Ok(DynamicFamily { - name: field(&family, "name")?.extract()?, - pivot_column: field(&family, "pivot_column")?.extract()?, - value_column: field(&family, "value_column")?.extract()?, - aggregate: field(&family, "aggregate")?.extract()?, - data_type: field(&family, "type")?.extract()?, - name_pattern: field(&family, "name_pattern")?.extract()?, - }) - }) - .collect() -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_metadata.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_metadata.rs index 068e2e8701..363dd967cc 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_metadata.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_metadata.rs @@ -1,7 +1,10 @@ //! Parsed MODEL header columns and audits built into the Python contract objects. +use crate::model_config::main::config::config_object::config_object; +use crate::model_config::main::headers::native_schema_column::native_schema_column; +use crate::model_config::main::headers::thresholds::thresholds; +use crate::model_config::main::model_nodes::contract_classes::contract_classes; use crate::model_config::models::ContractClasses; -use crate::model_config::models::HeaderColumn; use crate::model_config::models::NativeColumnSpec; use crate::model_config::types::HeaderValue; use std::sync::OnceLock; @@ -12,20 +15,14 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObject, IntoPyObjectExt}; use sqlbuild_discovery::model_files::models::{NativeAudit, NativeColumn}; use sqlbuild_discovery::models::LineColumnSpan; -use sqlbuild_model_config::header_metadata::models::{ - ParsedAudit, ParsedColumn, ParsedThresholds, ThresholdBound, -}; +use sqlbuild_model_config::header_metadata::models::{ParsedAudit, ParsedColumn, ThresholdBound}; use sqlbuild_sqltext::compiler::models::AuthoredValue; -use crate::model_config::_helpers::config::config_nodes::config_object; use crate::model_config::_helpers::config::typed_config::node_text; -use crate::model_config::_helpers::model_nodes::contract_classes::contract_classes; -use crate::model_config::models::HeaderAudits; use crate::model_config::models::PyNode; use crate::model_config::types::PyConfigNode; use crate::sqltext::main::authored_value_to_python::authored_value_to_python; use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::all_specs::all_specs; use sqlbuild_python_base::records::types::RecordId; @@ -111,22 +108,6 @@ pub(crate) enum ColumnFields { Projected, } -/// The `SchemaColumn` a native column projects to when its location names `location`'s path. -pub(crate) fn native_column_record<'py>( - py: Python<'py>, - column: &NativeColumn, - location: Option<(Bound<'py, PyAny>, LineColumnSpan)>, -) -> PyResult> { - let location: Bound<'py, PyAny> = match location { - Some((path, span)) => location_record(py, path, span)?, - None => py.None().into_bound(py), - }; - contract_object( - &record_class(py, RecordId::SchemaColumn)?, - column_values(py, column, location)?, - ) -} - /// A native column's record fields, in declaration order. pub(crate) fn native_values<'py>( py: Python<'py>, @@ -136,7 +117,7 @@ pub(crate) fn native_values<'py>( } /// A native column's record fields at `location`, in declaration order. -fn column_values<'py>( +pub(crate) fn column_values<'py>( py: Python<'py>, column: &NativeColumn, location: Bound<'py, PyAny>, @@ -187,7 +168,7 @@ pub(crate) fn native_location<'py>( } /// `SourceLocation(path, line, column, end_line, end_column)`. -fn location_record<'py>( +pub(crate) fn location_record<'py>( py: Python<'py>, path: Bound<'py, PyAny>, span: LineColumnSpan, @@ -281,14 +262,6 @@ pub(crate) fn schema_columns<'py>( Ok(columns) } -pub(crate) fn audit_tuple<'py>( - py: Python<'py>, - classes: &ContractClasses<'py>, - parsed: &[ParsedAudit>], -) -> PyResult> { - PyTuple::new(py, audit_instances(py, classes, parsed)?) -} - /// The parsed audits as `SchemaAuditInstance` objects without a location, in order. pub(crate) fn audit_instances<'py, N: HeaderValue>( py: Python<'py>, @@ -303,19 +276,6 @@ pub(crate) fn audit_instances<'py, N: HeaderValue>( Ok(audits) } -/// `audit.name or audit.definition_name` of the instance `audit` builds. -pub(crate) fn parsed_audit_identity( - py: Python<'_>, - audit: &ParsedAudit, -) -> PyResult { - if let Some(name) = &audit.name - && name.object(py)?.is_truthy()? - { - return name.exact_text(py); - } - audit.definition_name.exact_text(py) -} - /// `class(*values)`: the same record as keywords bind, given in `POSITIONAL_RECORDS` order. pub(crate) fn contract_object<'py, const N: usize>( class: &Bound<'py, PyAny>, @@ -358,7 +318,7 @@ fn mismatched_record() -> Option<&'static str> { None } -fn optional<'py, N: HeaderValue>( +pub(crate) fn optional<'py, N: HeaderValue>( py: Python<'py>, value: Option<&N>, ) -> PyResult> { @@ -377,43 +337,6 @@ pub(crate) fn schema_column<'py>( native_schema_column(py, classes, column, locations)?.project(py) } -pub(crate) fn native_schema_column<'py, N: HeaderValue>( - py: Python<'py>, - classes: &ContractClasses<'py>, - column: &ParsedColumn, - locations: &Bound<'py, PyDict>, -) -> PyResult { - let location = locations - .get_item(column.name.object(py)?)? - .unwrap_or_else(|| py.None().into_bound(py)); - let audits = HeaderAudits::new(py, classes, &column.audits, &location)?; - let projected_audits = if classes - .schema_column - .is(&record_class(py, RecordId::SchemaColumn)?) - { - PyTuple::empty(py) - } else { - audits.project(py)? - }; - let mut result = HeaderColumn::new( - &classes.schema_column, - [ - column.name.object(py)?, - optional(py, column.column_type.as_ref())?, - optional(py, column.nullable.as_ref())?, - optional(py, column.description.as_ref())?, - PyDict::new(py).into_any(), - projected_audits.into_any(), - location, - optional(py, column.migrate_from.as_ref())?, - ], - )?; - if !matches!(result.fields, ColumnFields::Projected) { - result.audits = Some(audits); - } - Ok(result) -} - pub(crate) fn audit_instance<'py, N: HeaderValue>( py: Python<'py>, classes: &ContractClasses<'py>, @@ -454,25 +377,7 @@ pub(crate) fn audit_instance<'py, N: HeaderValue>( ) } -/// `MeasurementThresholds` of `parsed`, or `None`. -pub(crate) fn thresholds<'py>( - py: Python<'py>, - classes: &ContractClasses<'py>, - parsed: Option<&ParsedThresholds>, -) -> PyResult> { - let Some(parsed) = parsed else { - return Ok(py.None().into_bound(py)); - }; - contract_object( - &classes.measurement_thresholds, - [ - threshold_bound(py, classes, parsed.warn)?, - threshold_bound(py, classes, parsed.error)?, - ], - ) -} - -fn threshold_bound<'py>( +pub(crate) fn threshold_bound<'py>( py: Python<'py>, classes: &ContractClasses<'py>, bound: Option, diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/mod.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/mod.rs index cf1bbd1e20..cf36afabaf 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/headers/mod.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/headers/mod.rs @@ -1,4 +1,3 @@ pub(crate) mod header_audits; pub(crate) mod header_columns; -pub(crate) mod header_families; pub(crate) mod header_metadata; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/column_migrations.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/column_migrations.rs index 41a2ea95ec..4c0fa7f06a 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/column_migrations.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/column_migrations.rs @@ -1,36 +1,4 @@ //! Column renames validated natively, raised by Python as the attachment raises them. -use pyo3::prelude::{Bound, PyAnyMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; -use sqlbuild_model_config::errors::ConfigError; -use sqlbuild_model_config::model_validation::main::validate_column_migrations::validate_column_migrations; -use sqlbuild_model_config::model_validation::models::ColumnMigration; - -const MATERIALIZED_KEY: &str = "materialized"; -const MIGRATE_FROM_KEY: &str = "migrate_from"; - -/// The first column-rename problem for a model with these config values, if any. -pub(crate) fn column_migration_error( - model_name: &str, - values: &Bound<'_, PyDict>, - columns: &[ColumnMigration], -) -> PyResult> { - if columns.iter().all(|column| column.migrate_from.is_none()) { - return Ok(None); - } - let materialized: Option = match values.get_item(MATERIALIZED_KEY)? { - None => None, - Some(value) if value.is_none() => None, - Some(value) => match value.downcast::() { - Ok(text) => Some(text.to_str()?.to_owned()), - Err(_) => return Ok(Some(ConfigError::config_value_type(MATERIALIZED_KEY))), - }, - }; - let model_migrates: bool = values - .get_item(MIGRATE_FROM_KEY)? - .is_some_and(|value| !value.is_none()); - Ok( - validate_column_migrations(model_name, materialized.as_deref(), model_migrates, columns) - .err(), - ) -} +pub(crate) const MATERIALIZED_KEY: &str = "materialized"; +pub(crate) const MIGRATE_FROM_KEY: &str = "migrate_from"; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs index 8666da7ad7..6a4c93a030 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs @@ -1,5 +1,6 @@ //! The Python classes native header and YAML parsing build contract objects as. +use crate::model_config::main::model_nodes::contract_classes::contract_classes; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods}; @@ -54,24 +55,10 @@ const INTEGRATION_ENTRY_CLASSES: [(&str, ThirdPartyHost); 3] = [ ("dlt_resource_config", ThirdPartyHost::DltResourceConfig), ]; -static CONTRACT_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); -static YAML_ENTRY_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); +pub(crate) static CONTRACT_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); +pub(crate) static YAML_ENTRY_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); -/// The contract classes native header parsing builds columns and audits as. -pub(crate) fn contract_classes(py: Python<'_>) -> PyResult> { - CONTRACT_CLASS_TABLE - .get_or_try_init(py, || built_contract_classes(py).map(Bound::unbind)) - .map(|classes| classes.bind(py).clone()) -} - -/// The classes parsed seed and source declarations become. -pub(crate) fn yaml_entry_classes(py: Python<'_>) -> PyResult> { - YAML_ENTRY_CLASS_TABLE - .get_or_try_init(py, || built_yaml_entry_classes(py).map(Bound::unbind)) - .map(|classes| classes.bind(py).clone()) -} - -fn built_contract_classes(py: Python<'_>) -> PyResult> { +pub(crate) fn built_contract_classes(py: Python<'_>) -> PyResult> { let classes: Bound<'_, PyDict> = PyDict::new(py); for (key, class) in CONTRACT_CLASSES { classes.set_item(key, record_class(py, class)?)?; @@ -87,7 +74,7 @@ fn built_contract_classes(py: Python<'_>) -> PyResult> { Ok(classes) } -fn built_yaml_entry_classes(py: Python<'_>) -> PyResult> { +pub(crate) fn built_yaml_entry_classes(py: Python<'_>) -> PyResult> { let classes: Bound<'_, PyDict> = PyDict::new(py); classes.set_item("contracts", contract_classes(py)?)?; for (key, class) in YAML_ENTRY_CLASSES { diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/described_models.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/described_models.rs index 4a1a1ef509..696a59cb1a 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/described_models.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/described_models.rs @@ -1,5 +1,8 @@ //! Which models `[defaults]`, `[path_defaults]` or a reusable model schema describe. +use crate::model_config::main::config::config_node::config_node; +use crate::model_config::main::config::config_object::config_object; +use crate::model_config::main::model_nodes::raised_config_error::raised_config_error; use std::collections::HashMap; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -11,12 +14,9 @@ use sqlbuild_model_config::path_defaults::main::select_path_default::select_path use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; use sqlbuild_model_config::types::AuthoredNode; -use crate::model_config::_helpers::config::config_nodes::PyConfigEntries; -use crate::model_config::_helpers::config::config_nodes::config_entries; -use crate::model_config::_helpers::config::config_nodes::config_node; -use crate::model_config::_helpers::config::config_nodes::config_object; -use crate::model_config::_helpers::config::config_nodes::is_text_node; -use crate::model_config::_helpers::model_nodes::model_config_builder::raised_config_error; +use crate::model_config::_helpers::config::config_nodes::{ + PyConfigEntries, config_entries, is_text_node, +}; use crate::model_config::_helpers::model_nodes::project_defaults::project_defaults_mapping; use crate::model_config::types::PyConfigNode; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_config_builder.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_config_builder.rs index bbf87cae21..00be9fcd29 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_config_builder.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_config_builder.rs @@ -1,12 +1,14 @@ //! The native model config build handed Python inputs once, and its results projected for Python. +use crate::model_config::main::config::carried_node::carried_node; +use crate::model_config::main::config::config_node::config_node; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::models::BuiltConfig; -use crate::model_config::models::NativeModelConfigBuilder; use std::collections::HashMap; use pyo3::PyErr; use pyo3::exceptions::{PyKeyError, PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; use sqlbuild_discovery::model_files::main::config_value::config_value; use sqlbuild_discovery::model_files::models::{EntryFacts, HeaderFacts}; @@ -20,16 +22,10 @@ use sqlbuild_model_config::model_config_build::types::ConfigBuildHost; use sqlbuild_sqltext::compiler::models::AuthoredValue; use crate::authored_values::main::project_var_text::project_var_text; -use crate::model_config::_helpers::config::config_errors::config_exception; -use crate::model_config::_helpers::config::config_errors::native_config_error; -use crate::model_config::_helpers::config::config_nodes::PyConfigEntries; -use crate::model_config::_helpers::config::config_nodes::carried_node; -use crate::model_config::_helpers::config::config_nodes::config_node; -use crate::model_config::_helpers::config::config_nodes::config_object; -use crate::model_config::_helpers::config::config_nodes::hook_entry; -use crate::model_config::_helpers::config::config_nodes::is_exact_container; +use crate::model_config::_helpers::config::config_nodes::{ + PyConfigEntries, hook_entry, is_exact_container, +}; use crate::model_config::_helpers::config::typed_config::built_values; -use crate::model_config::errors::NativeConfigError; use crate::model_config::models::HostHandle; use crate::model_config::types::PyConfigNode; @@ -256,20 +252,3 @@ fn projected<'py>( ) -> PyResult>> { value.map(|value| config_object(py, &value)).transpose() } - -/// The Python exception a native config error raises as. -pub(crate) fn raised_config_error(py: Python<'_>, error: ConfigError) -> PyErr { - let raised = || -> PyResult { - let error: Py = native_config_error(py, error)?; - Ok(PyErr::from_value(config_exception( - error.bind(py).as_any(), - None, - )?)) - }; - raised().unwrap_or_else(|error| error) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs index d90bce6055..f82f3a1eff 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs @@ -1,18 +1,17 @@ //! The Python model validators, run natively up to the first error they raise. -use crate::model_config::models::NativeModelValidator; +use crate::model_config::main::config::native_config_error::native_config_error; use sqlbuild_python_base::positional_record; use std::collections::HashSet; use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_model_config::model_validation::models::ValidationStop; use crate::discovery::main::project::column_name::column_name; use crate::discovery::main::project::each::each; use crate::discovery::main::project::stem::stem; -use crate::model_config::_helpers::config::config_errors::native_config_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; @@ -100,8 +99,3 @@ pub(crate) fn stop_object(py: Python<'_>, stop: ValidationStop) -> PyResult Ok(native_config_error(py, error)?.into_any()), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/unicode_environment.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/unicode_environment.rs index 76060f9163..df70b779e4 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/unicode_environment.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/unicode_environment.rs @@ -1,8 +1,6 @@ //! The process environment as compile text reads it, rejecting values that are not UTF-8. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyString}; use pyo3::{pyclass, pymethods}; @@ -68,12 +66,3 @@ impl UnicodeEnvironment { environ(py)?.len() } } - -/// A new view of the process environment as compile text reads it. -pub(crate) fn unicode_environment(py: Python<'_>) -> PyResult> { - Ok(Bound::new(py, UnicodeEnvironment)?.into_any()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() -} diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/carried_node.rs b/crates/sqlbuild-python-front/src/model_config/main/config/carried_node.rs index 4c32af982e..e30ee2aa49 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/carried_node.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/carried_node.rs @@ -1,13 +1,17 @@ //! Carry an uninspected value: exact containers keep their own objects without opening items. -use crate::model_config::_helpers::config::config_nodes; +use crate::model_config::_helpers::config::config_nodes::host; +use crate::model_config::_helpers::config::config_nodes::is_exact_container; +use crate::model_config::main::config::config_node::config_node; use crate::model_config::types::PyConfigNode; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_model_config::config_values::models::HostChildren; /// Carry an uninspected value: exact containers keep their own objects without opening items. -#[inline] pub fn carried_node(value: &Bound<'_, PyAny>) -> PyResult { - config_nodes::carried_node(value) + if is_exact_container(value) { + host(value, HostChildren::None) + } else { + config_node(value) + } } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_cursor_bound.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_cursor_bound.rs index d018ca76ee..fe5b28bad1 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_cursor_bound.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_cursor_bound.rs @@ -1,18 +1,23 @@ //! `key`'s cursor bound as text, or `None` when absent or `None`. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::cursor_bound_object; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::_helpers::config::typed_config::type_error; use crate::model_config::types::ConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_model_config::config_values::main::checked_cursor_bound::checked_cursor_bound; +use std::sync::Arc; /// `key`'s cursor bound as text, or `None` when absent or `None`. -#[inline] pub fn config_cursor_bound<'py>( py: Python<'py>, values: &ConfigValues<'_>, key: &str, ) -> PyResult>> { - typed_config::config_cursor_bound(py, values, key) + let node: Arc = node(values, key)?; + checked_cursor_bound(key, Some(&node)) + .map_err(|error| type_error(py, &error))? + .map(|bound| cursor_bound_object(py, bound)) + .transpose() } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_fields.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_fields.rs index 385beff58a..7a127f9563 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_fields.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_fields.rs @@ -1,18 +1,66 @@ //! Every `CompileModelConfig` field but `values`, recording the build's template reads first. -use crate::model_config::_helpers::config::config_classes; +use crate::model_config::_helpers::config::config_classes::PERMANENT_TABLE_TYPE; +use crate::model_config::_helpers::config::config_classes::TRANSIENT_TABLE_TYPE; +use crate::model_config::_helpers::config::config_classes::resolved_retention; +use crate::model_config::_helpers::config::config_classes::resolved_table_type; +use crate::model_config::_helpers::config::config_classes::storage_key; use crate::model_config::models::BuiltConfig; use crate::model_config::models::ConfigClasses; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_model_config::model_validation::models::{RetentionOverride, TableTypeOverride}; /// Every `CompileModelConfig` field but `values`, recording the build's template reads first. -#[inline] pub fn config_fields<'py>( classes: &ConfigClasses<'py>, built: &BuiltConfig<'py>, matched: Option<&str>, ) -> PyResult> { - config_classes::config_fields(classes, built, matched) + let py: Python<'py> = built.values.py(); + if !built.reads.is_empty() { + classes + .record_template_reads + .call1((PyTuple::new(py, &built.reads)?,))?; + } + let none: Bound<'py, PyAny> = py.None().into_bound(py); + let (inherited_retention, inherited_table_type): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + classes + .inherited_storage + .as_any() + .get_item(storage_key(classes, &built.values)?)? + .extract()?; + let retention: Bound<'py, PyAny> = match built.retention { + RetentionOverride::Inherit => inherited_retention, + RetentionOverride::Unmanaged => resolved_retention(classes, none.clone(), true)?, + RetentionOverride::Days(days) => { + resolved_retention(classes, days.into_pyobject(py)?.into_any(), false)? + } + }; + let table_type: Bound<'py, PyAny> = match built.table_type { + TableTypeOverride::Inherit => inherited_table_type, + TableTypeOverride::Permanent => resolved_table_type(classes, PERMANENT_TABLE_TYPE)?, + TableTypeOverride::Transient => resolved_table_type(classes, TRANSIENT_TABLE_TYPE)?, + }; + let logical_schema: Bound<'py, PyAny> = built.logical_schema.clone().unwrap_or(none.clone()); + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("model_header_keys", PyTuple::new(py, &built.header_keys)?)?; + fields.set_item("matched_path_default", matched)?; + fields.set_item("logical_schema", &logical_schema)?; + fields.set_item( + "layer_schema", + if built.layer_schema_configured { + logical_schema + } else { + none.clone() + }, + )?; + fields.set_item( + "logical_database", + built.logical_database.clone().unwrap_or(none), + )?; + fields.set_item("time_travel_retention", retention)?; + fields.set_item("table_type", table_type)?; + Ok(fields) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_flag.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_flag.rs index 6ae5cde364..5f091b71fd 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_flag.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_flag.rs @@ -1,12 +1,15 @@ //! `key`'s boolean value, or `None` when absent or `None`. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::_helpers::config::typed_config::type_error; use crate::model_config::types::ConfigValues; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_model_config::config_values::main::checked_flag::checked_flag; +use std::sync::Arc; /// `key`'s boolean value, or `None` when absent or `None`. -#[inline] pub fn config_flag(py: Python<'_>, values: &ConfigValues<'_>, key: &str) -> PyResult> { - typed_config::config_flag(py, values, key) + let node: Arc = node(values, key)?; + checked_flag(key, Some(&node)).map_err(|error| type_error(py, &error)) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_integer_object.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_integer_object.rs index a6ad072d82..27b8e6d8a1 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_integer_object.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_integer_object.rs @@ -1,18 +1,23 @@ //! `key`'s integer value as its Python object, or `None` when absent or `None`. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::_helpers::config::typed_config::type_error; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::types::ConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_model_config::config_values::main::checked_integer::checked_integer; +use std::sync::Arc; /// `key`'s integer value as its Python object, or `None` when absent or `None`. -#[inline] pub fn config_integer_object<'py>( py: Python<'py>, values: &ConfigValues<'_>, key: &str, ) -> PyResult>> { - typed_config::config_integer_object(py, values, key) + let node: Arc = node(values, key)?; + checked_integer(key, Some(&node)) + .map_err(|error| type_error(py, &error))? + .map(|value| config_object(py, value)) + .transpose() } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_node.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_node.rs index cfa8c0e2fd..7a1d0af107 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_node.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_node.rs @@ -1,13 +1,15 @@ //! One Python config value as a `ConfigNode`; a value that holds itself stays opaque whole. -use crate::model_config::_helpers::config::config_nodes; +use crate::model_config::_helpers::config::config_nodes::convert; +use crate::model_config::_helpers::config::config_nodes::host; use crate::model_config::types::PyConfigNode; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_model_config::config_values::models::HostChildren; /// One Python config value as a `ConfigNode`; a value that holds itself stays opaque whole. -#[inline] pub fn config_node(value: &Bound<'_, PyAny>) -> PyResult { - config_nodes::config_node(value) + match convert(value, &[])? { + Some(node) => Ok(node), + None => host(value, HostChildren::None), + } } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_node_of.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_node_of.rs index 8fa7b8d8f3..b43f99f24e 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_node_of.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_node_of.rs @@ -1,13 +1,12 @@ //! `values.get(key)` as its typed node, `None` (null) when absent. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; use crate::model_config::types::ConfigValues; use crate::model_config::types::PyConfigNode; use pyo3::prelude::PyResult; use std::sync::Arc; /// `values.get(key)` as its typed node, `None` (null) when absent. -#[inline] pub fn config_node_of(values: &ConfigValues<'_>, key: &str) -> PyResult> { - typed_config::config_node_of(values, key) + node(values, key) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_object.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_object.rs index dbe0dcfb9c..72c987410e 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_object.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_object.rs @@ -1,14 +1,37 @@ //! The Python object of one config value: new builtins, and opaque values as themselves. -use crate::model_config::_helpers::config::config_nodes; +use crate::model_config::_helpers::config::config_nodes::config_dict; use crate::model_config::types::PyConfigNode; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::{PyBool, PyFloat, PyList, PyString, PyTuple}; +use sqlbuild_model_config::config_values::models::ConfigNode; /// The Python object of one config value: new builtins, and opaque values as themselves. -#[inline] pub fn config_object<'py>(py: Python<'py>, node: &PyConfigNode) -> PyResult> { - config_nodes::config_object(py, node) + Ok(match node { + ConfigNode::Null => py.None().into_bound(py), + ConfigNode::Bool(flag) => PyBool::new(py, *flag).to_owned().into_any(), + ConfigNode::Int(number) => number.into_pyobject(py)?.into_any(), + ConfigNode::Float(number) => PyFloat::new(py, *number).into_any(), + ConfigNode::Str(text) => PyString::new(py, text).into_any(), + ConfigNode::List(items) => PyList::new( + py, + items + .iter() + .map(|item| config_object(py, item)) + .collect::>>()?, + )? + .into_any(), + ConfigNode::Tuple(items) => PyTuple::new( + py, + items + .iter() + .map(|item| config_object(py, item)) + .collect::>>()?, + )? + .into_any(), + ConfigNode::Map(entries) => config_dict(py, entries)?.into_any(), + ConfigNode::Host(host) => host.handle.0.bind(py).clone(), + }) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_str.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_str.rs index 2e2bf43ad3..9b6d05b42f 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_str.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_str.rs @@ -1,15 +1,20 @@ //! An authored string, preserving a present empty string. -use crate::model_config::_helpers::config::config_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::config::config_values::authored_node; +use crate::model_config::_helpers::config::typed_config::type_error; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_model_config::config_values::main::checked_text::checked_text; /// An authored string, preserving a present empty string. -#[inline] +#[pyfunction(name = "get_config_str")] +#[pyo3(signature = (*, values, key))] pub fn config_str<'py>( values: &Bound<'py, PyAny>, key: &str, ) -> PyResult>> { - config_values::config_str(values, key) + let (node, value) = authored_node(values, key)?; + let checked = + checked_text(key, Some(&node)).map_err(|error| type_error(values.py(), &error))?; + Ok(checked.map(|_| value)) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_text.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_text.rs index 50d70bbe63..e2222f3321 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_text.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_text.rs @@ -1,16 +1,23 @@ //! `key`'s string value, or `None` when absent or `None`. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::_helpers::config::typed_config::node_text; +use crate::model_config::_helpers::config::typed_config::type_error; use crate::model_config::types::ConfigValues; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_model_config::config_values::main::checked_text::checked_text; +use std::sync::Arc; /// `key`'s string value, or `None` when absent or `None`. -#[inline] pub fn config_text( py: Python<'_>, values: &ConfigValues<'_>, key: &str, ) -> PyResult> { - typed_config::config_text(py, values, key) + let node: Arc = node(values, key)?; + checked_text(key, Some(&node)) + .map_err(|error| type_error(py, &error))? + .map(|value| node_text(py, value)) + .transpose() } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_text_is.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_is.rs index 44ca9d53fe..2f8f7d5366 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_text_is.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_is.rs @@ -1,17 +1,29 @@ //! Whether `key` holds a string equal to `expected`, as Python's `==` compares them. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::_helpers::config::typed_config::type_error; use crate::model_config::types::ConfigValues; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_model_config::config_values::main::checked_text::checked_text; +use sqlbuild_model_config::config_values::models::ConfigNode; +use std::sync::Arc; /// Whether `key` holds a string equal to `expected`, as Python's `==` compares them. -#[inline] pub fn config_text_is( py: Python<'_>, values: &ConfigValues<'_>, key: &str, expected: &str, ) -> PyResult { - typed_config::config_text_is(py, values, key, expected) + let node: Arc = node(values, key)?; + let text: Option<&PyConfigNode> = + checked_text(key, Some(&node)).map_err(|error| type_error(py, &error))?; + Ok(match text { + Some(ConfigNode::Str(text)) => text == expected, + Some(ConfigNode::Host(host)) if host.facts.strict_text => { + host.facts.text.as_deref() == Some(expected) + } + _ => false, + }) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_text_object.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_object.rs index 21b9eb3c03..4ed04a64c5 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_text_object.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_object.rs @@ -1,18 +1,23 @@ //! `key`'s string value as its Python object, or `None` when absent or `None`. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::_helpers::config::typed_config::type_error; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::types::ConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_model_config::config_values::main::checked_text::checked_text; +use std::sync::Arc; /// `key`'s string value as its Python object, or `None` when absent or `None`. -#[inline] pub fn config_text_object<'py>( py: Python<'py>, values: &ConfigValues<'_>, key: &str, ) -> PyResult>> { - typed_config::config_text_object(py, values, key) + let node: Arc = node(values, key)?; + checked_text(key, Some(&node)) + .map_err(|error| type_error(py, &error))? + .map(|value| config_object(py, value)) + .transpose() } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_text_tuple.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_tuple.rs index 97dfd82467..73bf247ace 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_text_tuple.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_text_tuple.rs @@ -1,18 +1,29 @@ //! `key`'s strings as a tuple, or `None` when absent or `None`. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::_helpers::config::typed_config::type_error; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::types::ConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyTuple; +use sqlbuild_model_config::config_values::main::checked_texts::checked_texts; +use std::sync::Arc; /// `key`'s strings as a tuple, or `None` when absent or `None`. -#[inline] pub fn config_text_tuple<'py>( py: Python<'py>, values: &ConfigValues<'_>, key: &str, ) -> PyResult>> { - typed_config::config_text_tuple(py, values, key) + let node: Arc = node(values, key)?; + let Some(items) = checked_texts(key, Some(&node)).map_err(|error| type_error(py, &error))? + else { + return Ok(None); + }; + let objects: Vec> = items + .into_iter() + .map(|item| config_object(py, item)) + .collect::>()?; + PyTuple::new(py, objects).map(Some) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/config_value.rs b/crates/sqlbuild-python-front/src/model_config/main/config/config_value.rs index dac00dad84..1d2df8bd45 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/config_value.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/config_value.rs @@ -1,18 +1,18 @@ //! `values.get(key)` as its Python object, `None` when absent. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::node; +use crate::model_config::main::config::config_object::config_object; use crate::model_config::types::ConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::types::PyConfigNode; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use std::sync::Arc; /// `values.get(key)` as its Python object, `None` when absent. -#[inline] pub fn config_value<'py>( py: Python<'py>, values: &ConfigValues<'_>, key: &str, ) -> PyResult> { - typed_config::config_value(py, values, key) + let value: Arc = node(values, key)?; + config_object(py, &value) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/early_model_templates.rs b/crates/sqlbuild-python-front/src/model_config/main/config/early_model_templates.rs index aa4ac4f1f3..9c976302b0 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/early_model_templates.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/early_model_templates.rs @@ -1,16 +1,33 @@ //! Model header `values` with `${name}`, `${ENV:...}` and early `run.*` templates resolved. -use crate::model_config::_helpers::config::template_data; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::config::template_data::MODEL_CONFIG_LABEL; +use crate::model_config::_helpers::config::template_data::RUN_ID_KEY; +use crate::model_config::_helpers::config::template_data::RUN_TARGET_KEY; +use crate::model_config::main::config::expanded_template_data::expanded_template_data; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_model_config::templates::models::TemplateOptions; /// Model header `values` with `${name}`, `${ENV:...}` and early `run.*` templates resolved. -#[inline] pub fn early_model_templates<'py>( values: &Bound<'py, PyAny>, effective_vars: &Bound<'py, PyAny>, (target_name, run_id): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { - template_data::early_model_templates(values, effective_vars, (target_name, run_id)) + let py: Python<'py> = values.py(); + let context_values: Bound<'py, PyDict> = PyDict::new(py); + context_values.set_item(RUN_ID_KEY, run_id)?; + context_values.set_item(RUN_TARGET_KEY, target_name)?; + expanded_template_data( + values, + (effective_vars.downcast()?, &context_values), + ( + TemplateOptions { + allow_context: true, + preserve_context_tokens: false, + preserve_unknown_context: true, + }, + MODEL_CONFIG_LABEL, + ), + ) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/expanded.rs b/crates/sqlbuild-python-front/src/model_config/main/config/expanded.rs index 77bd879839..d1e35728eb 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/expanded.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/expanded.rs @@ -1,18 +1,61 @@ //! Expand every template in `value` and the containers inside it. -use crate::model_config::_helpers::config::config_templates; +use crate::model_config::_helpers::config::config_templates::TEMPLATE_OPEN_TOKEN; use crate::model_config::models::PythonHost; use crate::model_config::models::Stop; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use sqlbuild_model_config::templates::models::TemplateOptions; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods}; +use pyo3::types::{ + PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, + PyTupleMethods, +}; +use sqlbuild_model_config::templates::main::expand_template_string::expand_template_string; +use sqlbuild_model_config::templates::models::{StringExpansion, TemplateOptions}; /// Expand every template in `value` and the containers inside it. -#[inline] pub fn expanded<'py>( host: &PythonHost<'py>, value: &Bound<'py, PyAny>, options: TemplateOptions, ) -> Result, Stop> { - config_templates::expanded(host, value, options) + let py = host.py; + if let Ok(text) = value.downcast::() { + let text = text.to_str().map_err(Stop::Python)?; + if !text.contains(TEMPLATE_OPEN_TOKEN) { + return Ok(value.clone()); + } + return match expand_template_string(host, text, options) { + Ok(StringExpansion::Unchanged) => Ok(PyString::new(py, text).into_any()), + Ok(StringExpansion::Value(result)) => Ok(result), + Ok(StringExpansion::Text(result)) => Ok(PyString::new(py, &result).into_any()), + Err(failure) => Err(host.stop(failure)), + }; + } + if let Ok(mapping) = value.downcast::() { + let result = PyDict::new(py); + for (key, item) in mapping.iter() { + result + .set_item(key, expanded(host, &item, options)?) + .map_err(Stop::Python)?; + } + return Ok(result.into_any()); + } + if let Ok(items) = value.downcast::() { + let result = items + .iter() + .map(|item| expanded(host, &item, options)) + .collect::, _>>()?; + return PyList::new(py, result) + .map(Bound::into_any) + .map_err(Stop::Python); + } + if let Ok(items) = value.downcast::() { + let result = items + .iter() + .map(|item| expanded(host, &item, options)) + .collect::, _>>()?; + return PyTuple::new(py, result) + .map(Bound::into_any) + .map_err(Stop::Python); + } + Ok(value.clone()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/expanded_effective_vars.rs b/crates/sqlbuild-python-front/src/model_config/main/config/expanded_effective_vars.rs index 4ad1859ae5..7f20b1a239 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/expanded_effective_vars.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/expanded_effective_vars.rs @@ -1,15 +1,46 @@ //! Merged effective vars with recursive `${name}` expansion. -use crate::model_config::_helpers::config::template_data; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use crate::model_config::_helpers::config::config_templates::{ + lone_surrogate_site, resolve_effective_vars, +}; +use crate::model_config::_helpers::config::template_data::applied; +use crate::model_config::_helpers::config::template_data::environment; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyString}; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; /// Merged effective vars with recursive `${name}` expansion. -#[inline] pub fn expanded_effective_vars<'py>( raw_values: &Bound<'py, PyDict>, ) -> PyResult> { - template_data::expanded_effective_vars(raw_values) + let py: Python<'py> = raw_values.py(); + if let Some((name, path, byte_offset)) = lone_surrogate_site(raw_values)? { + let error: Bound<'py, PyAny> = error_class(py, ErrorId::CompileInput)?.call( + (format!( + "Variable '{name}' holds a lone surrogate in {path} at UTF-8 byte \ + {byte_offset}, which is not valid Unicode text" + ),), + Some(&keywords( + py, + &[( + "help", + PyString::new( + py, + &format!( + "Set '{name}' to valid Unicode text in --vars or the [vars] \ + table; the value is not shown because it may be a secret" + ), + ) + .into_any(), + )], + )?), + )?; + return Err(PyErr::from_value(error)); + } + applied( + py, + resolve_effective_vars(py, raw_values, environment(py)?)?, + ) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/expanded_template_data.rs b/crates/sqlbuild-python-front/src/model_config/main/config/expanded_template_data.rs index d4d335fff6..fdfbe56e1b 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/expanded_template_data.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/expanded_template_data.rs @@ -1,18 +1,23 @@ //! `value` with every `${...}` template expanded, recording the reads it made. -use crate::model_config::_helpers::config::template_data; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::config::config_templates::expand_config_templates; +use crate::model_config::_helpers::config::template_data::applied; +use crate::model_config::_helpers::config::template_data::environment; +use crate::model_config::models::TemplateSources; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyDict; use sqlbuild_model_config::templates::models::TemplateOptions; /// `value` with every `${...}` template expanded, recording the reads it made. -#[inline] pub fn expanded_template_data<'py>( value: &Bound<'py, PyAny>, (variables, context_values): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), (options, label): (TemplateOptions, &str), ) -> PyResult> { - template_data::expanded_template_data(value, (variables, context_values), (options, label)) + let py: Python<'py> = value.py(); + let sources = TemplateSources(variables.clone(), environment(py)?, context_values.clone()); + applied( + py, + expand_config_templates(py, value, sources, (options, label))?, + ) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/model_values.rs b/crates/sqlbuild-python-front/src/model_config/main/config/model_values.rs index f8a5513f46..0562f0ecc6 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/model_values.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/model_values.rs @@ -1,13 +1,12 @@ //! A model's config values as typed nodes, read key by key. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::CONFIG_FIELD; +use crate::model_config::main::config::typed_values::typed_values; use crate::model_config::types::ConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// A model's config values as typed nodes, read key by key. -#[inline] pub fn model_values<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - typed_config::model_values(model) + typed_values(&native_field(model, CONFIG_FIELD)?) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/native_config_error.rs b/crates/sqlbuild-python-front/src/model_config/main/config/native_config_error.rs index ecd8bf24c2..3552e4a3e7 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/native_config_error.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/native_config_error.rs @@ -1,14 +1,30 @@ //! Wrap `error` for Python. -use crate::model_config::_helpers::config::config_errors; +use crate::model_config::_helpers::config::config_errors::COMPILE_INPUT_CLASS; +use crate::model_config::_helpers::config::config_errors::CONFIG_VALUE_TYPE_CLASS; +use crate::model_config::_helpers::config::config_errors::DISCOVERY_CONFLICT_CLASS; +use crate::model_config::_helpers::config::config_errors::RESOURCE_IDENTITY_CLASS; use crate::model_config::errors::NativeConfigError; use pyo3::Py; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use sqlbuild_model_config::errors::ConfigError; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; /// Wrap `error` for Python. -#[inline] pub fn native_config_error(py: Python<'_>, error: ConfigError) -> PyResult> { - config_errors::native_config_error(py, error) + let class_name = match error.class { + ErrorClass::CompileInput => COMPILE_INPUT_CLASS, + ErrorClass::ConfigValueType => CONFIG_VALUE_TYPE_CLASS, + ErrorClass::ResourceIdentity => RESOURCE_IDENTITY_CLASS, + ErrorClass::DiscoveryConflict => DISCOVERY_CONFLICT_CLASS, + }; + Py::new( + py, + NativeConfigError { + class_name, + message: error.message, + code: error.code, + help: error.help, + key: error.key, + }, + ) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/raised_config_exception.rs b/crates/sqlbuild-python-front/src/model_config/main/config/raised_config_exception.rs index a9c52c7bc0..9a590b7d43 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/raised_config_exception.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/raised_config_exception.rs @@ -1,15 +1,16 @@ //! `config_exception` as the raised error, or the failure building it. -use crate::model_config::_helpers::config::config_errors; +use crate::model_config::_helpers::config::config_errors::config_exception; use pyo3::PyErr; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; +use pyo3::prelude::{Bound, PyAny}; /// `config_exception` as the raised error, or the failure building it. -#[inline] pub fn raised_config_exception( error: &Bound<'_, PyAny>, values: Option<&Bound<'_, PyAny>>, ) -> PyErr { - config_errors::raised_config_exception(error, values) + match config_exception(error, values) { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/record_reads.rs b/crates/sqlbuild-python-front/src/model_config/main/config/record_reads.rs index b45e88ec17..ef6515c5da 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/record_reads.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/record_reads.rs @@ -1,14 +1,21 @@ //! Record native template environment and context reads for project reuse, in order. -use crate::model_config::_helpers::config::template_data; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::_helpers::config::template_data::ENVIRONMENT_READ; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_python_base::compile_context::main::input_reads::context_read::context_read; +use sqlbuild_python_base::compile_context::main::input_reads::environment_read::environment_read; /// Record native template environment and context reads for project reuse, in order. -#[inline] pub fn record_reads<'a>( py: Python<'_>, reads: impl IntoIterator, ) -> PyResult<()> { - template_data::record_reads(py, reads) + for (kind, name) in reads { + if kind == ENVIRONMENT_READ { + environment_read(py, name)?; + } else { + context_read(py, name)?; + } + } + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/record_template_reads_function.rs b/crates/sqlbuild-python-front/src/model_config/main/config/record_template_reads_function.rs index 975be395fa..5916330ff0 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/record_template_reads_function.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/record_template_reads_function.rs @@ -1,11 +1,10 @@ //! `record_template_reads` as a Python function. -use crate::model_config::_helpers::config::template_data; +use crate::model_config::_helpers::config::template_data::record_template_reads; /// `record_template_reads` as a Python function. -#[inline] pub fn record_template_reads_function( py: pyo3::Python<'_>, ) -> pyo3::PyResult> { - template_data::record_template_reads_function(py) + pyo3::wrap_pyfunction!(record_template_reads, py) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/register_config_errors.rs b/crates/sqlbuild-python-front/src/model_config/main/config/register_config_errors.rs index bfb2b4dff2..302aea46a9 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/register_config_errors.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/register_config_errors.rs @@ -1,12 +1,10 @@ //! Register the config errors functions and classes. -use crate::model_config::_helpers::config::config_errors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::model_config::errors::NativeConfigError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the config errors functions and classes. -#[inline] pub fn register_config_errors(module: &Bound<'_, PyModule>) -> PyResult<()> { - config_errors::register(module) + module.add_class::()?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs b/crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs index 209cb3c148..0b1daefb3a 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs @@ -1,12 +1,19 @@ //! Register the config values functions and classes. -use crate::model_config::_helpers::config::config_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::config::config_values::config_bool; +use crate::model_config::_helpers::config::config_values::config_cursor_bound; +use crate::model_config::_helpers::config::config_values::config_int; +use crate::model_config::_helpers::config::config_values::config_string_tuple; +use crate::model_config::main::config::config_str::config_str; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the config values functions and classes. -#[inline] pub fn register_config_values(module: &Bound<'_, PyModule>) -> PyResult<()> { - config_values::register(module) + module.add_function(wrap_pyfunction!(config_str, module)?)?; + module.add_function(wrap_pyfunction!(config_bool, module)?)?; + module.add_function(wrap_pyfunction!(config_string_tuple, module)?)?; + module.add_function(wrap_pyfunction!(config_int, module)?)?; + module.add_function(wrap_pyfunction!(config_cursor_bound, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs b/crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs index 916f95a334..cb44476667 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs @@ -1,12 +1,17 @@ //! Register the template data functions and classes. -use crate::model_config::_helpers::config::template_data; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::config::template_data::expand_effective_vars; +use crate::model_config::_helpers::config::template_data::expand_template_data; +use crate::model_config::_helpers::config::template_data::record_template_reads; +use crate::model_config::_helpers::config::template_data::resolve_run_id; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the template data functions and classes. -#[inline] pub fn register_template_data(module: &Bound<'_, PyModule>) -> PyResult<()> { - template_data::register(module) + module.add_function(wrap_pyfunction!(expand_template_data, module)?)?; + module.add_function(wrap_pyfunction!(expand_effective_vars, module)?)?; + module.add_function(wrap_pyfunction!(record_template_reads, module)?)?; + module.add_function(wrap_pyfunction!(resolve_run_id, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/retain_values.rs b/crates/sqlbuild-python-front/src/model_config/main/config/retain_values.rs index 22552f8b28..189ac6700a 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/retain_values.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/retain_values.rs @@ -1,13 +1,26 @@ //! Attach the built tree to its public record, rather than reconstructing it from that view. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::TYPED_VALUES_SLOT; +use crate::model_config::models::TypedConfigValues; use crate::model_config::types::BuiltConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::store_cached::store_cached; +use sqlbuild_python_base::records::models::NativeRecord; +use std::sync::{Arc, Mutex}; /// Attach the built tree to its public record, rather than reconstructing it from that view. -#[inline] pub fn retain_values(config: &Bound<'_, PyAny>, built: Option<&BuiltConfigValues>) -> PyResult<()> { - typed_config::retain_values(config, built) + if let Some(built) = built + && let Ok(record) = config.cast::() + { + let typed = Bound::new( + config.py(), + TypedConfigValues { + built: Some(Arc::clone(built)), + read: Mutex::new(Vec::new()), + }, + )?; + store_cached(record, TYPED_VALUES_SLOT, &typed.into_any().unbind())?; + } + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/run_id.rs b/crates/sqlbuild-python-front/src/model_config/main/config/run_id.rs index c10fda69f6..e2f112aba9 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/run_id.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/run_id.rs @@ -1,12 +1,30 @@ //! The selected run id, or a new `YYYYMMDDTHHMMSSZ_<12 hex>` compile invocation id. -use crate::model_config::_helpers::config::template_data; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::config::template_data::RUN_ID_SUFFIX_LENGTH; +use crate::model_config::_helpers::config::template_data::TIMESTAMP_SEPARATORS; +use crate::model_config::_helpers::config::template_data::UUID_MODULE; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_core::scenario_command::main::format_captured_at::format_captured_at; +use sqlbuild_python_base::boundary::main::imported::imported; +use std::time::{SystemTime, UNIX_EPOCH}; /// The selected run id, or a new `YYYYMMDDTHHMMSSZ_<12 hex>` compile invocation id. -#[inline] pub fn run_id<'py>(selected_run_id: &Bound<'py, PyAny>) -> PyResult> { - template_data::run_id(selected_run_id) + let py: Python<'py> = selected_run_id.py(); + if !selected_run_id.is_none() { + return Ok(selected_run_id.clone()); + } + let epoch_seconds: i64 = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX) + }); + let timestamp: String = format_captured_at(epoch_seconds).replace(TIMESTAMP_SEPARATORS, ""); + let suffix: String = imported(py, UUID_MODULE, "uuid4")? + .call0()? + .getattr("hex")? + .extract()?; + let suffix: &str = suffix.get(..RUN_ID_SUFFIX_LENGTH).unwrap_or(&suffix); + Ok(PyString::new(py, &format!("{timestamp}_{suffix}")).into_any()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/template_error.rs b/crates/sqlbuild-python-front/src/model_config/main/config/template_error.rs index edc8f6f2a5..de6b631cb2 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/template_error.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/template_error.rs @@ -1,11 +1,10 @@ //! The `CompileInputError` Python raises for a template error in the context `label`. -use crate::model_config::_helpers::config::config_templates; use sqlbuild_model_config::errors::ConfigError; use sqlbuild_model_config::templates::errors::TemplateError; +use sqlbuild_model_config::templates::main::template_error_message::template_error_message; /// The `CompileInputError` Python raises for a template error in the context `label`. -#[inline] pub fn template_error(error: &TemplateError, label: &str) -> ConfigError { - config_templates::template_error(error, label) + ConfigError::compile(template_error_message(error, label)) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/typed_values.rs b/crates/sqlbuild-python-front/src/model_config/main/config/typed_values.rs index ca4d969669..63ff6d8a02 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/typed_values.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/typed_values.rs @@ -1,13 +1,44 @@ //! A config record's values, their typed nodes kept on the record so each key converts once. -use crate::model_config::_helpers::config::typed_config; +use crate::model_config::_helpers::config::typed_config::TYPED_VALUES_SLOT; +use crate::model_config::_helpers::config::typed_config::VALUES_FIELD; +use crate::model_config::models::TypedConfigValues; use crate::model_config::types::ConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::cached_value::cached_value; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::store_cached::store_cached; +use sqlbuild_python_base::records::models::NativeRecord; +use std::sync::Mutex; /// A config record's values, their typed nodes kept on the record so each key converts once. -#[inline] pub fn typed_values<'py>(config: &Bound<'py, PyAny>) -> PyResult> { - typed_config::typed_values(config) + let py: Python<'py> = config.py(); + let values: Bound<'py, PyAny> = native_field(config, VALUES_FIELD)?; + let fresh = || { + Bound::new( + py, + TypedConfigValues { + built: None, + read: Mutex::new(Vec::new()), + }, + ) + }; + let Ok(record) = config.cast::() else { + let typed: Bound<'py, TypedConfigValues> = fresh()?; + return Ok((values, typed)); + }; + let typed: Bound<'py, TypedConfigValues> = match cached_value(record, TYPED_VALUES_SLOT)? { + Some(typed) => typed.into_bound(py).cast_into()?, + None => { + let typed: Bound<'py, TypedConfigValues> = fresh()?; + store_cached( + record, + TYPED_VALUES_SLOT, + &typed.clone().into_any().unbind(), + )?; + typed + } + }; + Ok((values, typed)) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/config/view.rs b/crates/sqlbuild-python-front/src/model_config/main/config/view.rs index 2f6ed7ebdd..e9eb61520e 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/config/view.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/config/view.rs @@ -1,20 +1,22 @@ //! A `CompileModelConfig` of `values` and the other `fields`. -use crate::model_config::_helpers::config::config_classes; +use crate::model_config::_helpers::config::config_classes::VALUES_FIELD; +use crate::model_config::main::config::retain_values::retain_values; use crate::model_config::models::ConfigClasses; use crate::model_config::types::BuiltConfigValues; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; /// A `CompileModelConfig` of `values` and the other `fields`. -#[inline] pub fn view<'py>( classes: &ConfigClasses<'py>, values: &Bound<'py, PyDict>, fields: &Bound<'py, PyDict>, nodes: Option<&BuiltConfigValues>, ) -> PyResult> { - config_classes::view(classes, values, fields, nodes) + let arguments: Bound<'py, PyDict> = fields.copy()?; + arguments.set_item(VALUES_FIELD, values)?; + let config = classes.compile_model_config.call((), Some(&arguments))?; + retain_values(&config, nodes)?; + Ok(config) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/audit_tuple.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/audit_tuple.rs index b933a5c862..f234b07fcf 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/audit_tuple.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/audit_tuple.rs @@ -1,20 +1,17 @@ //! Audit tuple. -use crate::model_config::_helpers::headers::header_metadata; +use crate::model_config::_helpers::headers::header_metadata::audit_instances; use crate::model_config::models::ContractClasses; use crate::model_config::models::PyNode; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyResult, Python}; use pyo3::types::PyTuple; use sqlbuild_model_config::header_metadata::models::ParsedAudit; /// Audit tuple. -#[inline] pub fn audit_tuple<'py>( py: Python<'py>, classes: &ContractClasses<'py>, parsed: &[ParsedAudit>], ) -> PyResult> { - header_metadata::audit_tuple(py, classes, parsed) + PyTuple::new(py, audit_instances(py, classes, parsed)?) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs index 040da859c0..510291ef68 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs @@ -1,13 +1,23 @@ //! A Python-authored family contract crosses into native ownership once. -use crate::model_config::_helpers::headers::header_families; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_model_config::header_metadata::models::DynamicFamily; +use sqlbuild_python_base::records::main::fields::field::field; /// A Python-authored family contract crosses into native ownership once. -#[inline] pub fn family_specs(families: &Bound<'_, PyAny>) -> PyResult> { - header_families::family_specs(families) + families + .try_iter()? + .map(|family| { + let family = family?; + Ok(DynamicFamily { + name: field(&family, "name")?.extract()?, + pivot_column: field(&family, "pivot_column")?.extract()?, + value_column: field(&family, "value_column")?.extract()?, + aggregate: field(&family, "aggregate")?.extract()?, + data_type: field(&family, "type")?.extract()?, + name_pattern: field(&family, "name_pattern")?.extract()?, + }) + }) + .collect() } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/native_column_record.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/native_column_record.rs index 78e06cd911..43b13afaae 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/native_column_record.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/native_column_record.rs @@ -1,19 +1,26 @@ //! The `SchemaColumn` a native column projects to when its location names `location`'s path. -use crate::model_config::_helpers::headers::header_metadata; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::_helpers::headers::header_metadata::column_values; +use crate::model_config::_helpers::headers::header_metadata::contract_object; +use crate::model_config::_helpers::headers::header_metadata::location_record; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use sqlbuild_discovery::model_files::models::NativeColumn; use sqlbuild_discovery::models::LineColumnSpan; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::types::RecordId; /// The `SchemaColumn` a native column projects to when its location names `location`'s path. -#[inline] pub fn native_column_record<'py>( py: Python<'py>, column: &NativeColumn, location: Option<(Bound<'py, PyAny>, LineColumnSpan)>, ) -> PyResult> { - header_metadata::native_column_record(py, column, location) + let location: Bound<'py, PyAny> = match location { + Some((path, span)) => location_record(py, path, span)?, + None => py.None().into_bound(py), + }; + contract_object( + &record_class(py, RecordId::SchemaColumn)?, + column_values(py, column, location)?, + ) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/native_schema_column.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/native_schema_column.rs index 5fe9654ff2..0810369a23 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/native_schema_column.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/native_schema_column.rs @@ -1,22 +1,51 @@ //! Native schema column. -use crate::model_config::_helpers::headers::header_metadata; +use crate::model_config::_helpers::headers::header_metadata::ColumnFields; +use crate::model_config::_helpers::headers::header_metadata::optional; use crate::model_config::models::ContractClasses; +use crate::model_config::models::HeaderAudits; use crate::model_config::models::HeaderColumn; use crate::model_config::types::HeaderValue; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyDict; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_model_config::header_metadata::models::ParsedColumn; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::types::RecordId; /// Native schema column. -#[inline] pub fn native_schema_column<'py, N: HeaderValue>( py: Python<'py>, classes: &ContractClasses<'py>, column: &ParsedColumn, locations: &Bound<'py, PyDict>, ) -> PyResult { - header_metadata::native_schema_column(py, classes, column, locations) + let location = locations + .get_item(column.name.object(py)?)? + .unwrap_or_else(|| py.None().into_bound(py)); + let audits = HeaderAudits::new(py, classes, &column.audits, &location)?; + let projected_audits = if classes + .schema_column + .is(&record_class(py, RecordId::SchemaColumn)?) + { + PyTuple::empty(py) + } else { + audits.project(py)? + }; + let mut result = HeaderColumn::new( + &classes.schema_column, + [ + column.name.object(py)?, + optional(py, column.column_type.as_ref())?, + optional(py, column.nullable.as_ref())?, + optional(py, column.description.as_ref())?, + PyDict::new(py).into_any(), + projected_audits.into_any(), + location, + optional(py, column.migrate_from.as_ref())?, + ], + )?; + if !matches!(result.fields, ColumnFields::Projected) { + result.audits = Some(audits); + } + Ok(result) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/parsed_audit_identity.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/parsed_audit_identity.rs index aa2fc6838a..a8223d0eb2 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/parsed_audit_identity.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/parsed_audit_identity.rs @@ -1,16 +1,18 @@ //! `audit.name or audit.definition_name` of the instance `audit` builds. -use crate::model_config::_helpers::headers::header_metadata; use crate::model_config::types::HeaderValue; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{PyAnyMethods, PyResult, Python}; use sqlbuild_model_config::header_metadata::models::ParsedAudit; /// `audit.name or audit.definition_name` of the instance `audit` builds. -#[inline] pub fn parsed_audit_identity( py: Python<'_>, audit: &ParsedAudit, ) -> PyResult { - header_metadata::parsed_audit_identity(py, audit) + if let Some(name) = &audit.name + && name.object(py)?.is_truthy()? + { + return name.exact_text(py); + } + audit.definition_name.exact_text(py) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/register_header_columns.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/register_header_columns.rs index 398b17e024..af24bfc037 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/register_header_columns.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/register_header_columns.rs @@ -1,12 +1,10 @@ //! Register the header columns functions and classes. -use crate::model_config::_helpers::headers::header_columns; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::headers::header_columns::parse_schema_columns; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the header columns functions and classes. -#[inline] pub fn register_header_columns(module: &Bound<'_, PyModule>) -> PyResult<()> { - header_columns::register(module) + module.add_function(wrap_pyfunction!(parse_schema_columns, module)?) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/schema_columns.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/schema_columns.rs index 6c5318feff..8a6a90a109 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/schema_columns.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/schema_columns.rs @@ -1,21 +1,35 @@ //! The `SchemaColumn` tuple of one authored columns mapping, raising Python's error for it. -use crate::model_config::_helpers::headers::header_columns; use crate::model_config::_helpers::headers::header_columns::ColumnOptions; use crate::model_config::_helpers::headers::header_columns::ColumnSite; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use crate::model_config::_helpers::headers::header_columns::raised; +use crate::model_config::_helpers::headers::header_metadata::column_tuple; +use crate::model_config::main::model_nodes::contract_classes::contract_classes; +use crate::model_config::models::ContractClasses; +use crate::model_config::models::PyNode; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_model_config::header_metadata::main::parse_schema_columns::parse_schema_columns as parse_columns; +use sqlbuild_model_config::header_metadata::models::ColumnRequest; /// The `SchemaColumn` tuple of one authored columns mapping, raising Python's error for it. -#[inline] pub fn schema_columns<'py>( raw_columns: &Bound<'py, PyAny>, site: ColumnSite<'_>, options: ColumnOptions, column_locations: &Bound<'py, PyDict>, ) -> PyResult> { - header_columns::schema_columns(raw_columns, site, options, column_locations) + let py: Python<'py> = raw_columns.py(); + let (path, label) = site; + let (require_columns, allow_migrate_from, declaration_errors) = options; + let request = ColumnRequest { + path, + label, + require_columns, + allow_migrate_from, + }; + let columns = parse_columns(&PyNode(raw_columns.clone()), request) + .map_err(|error| raised(py, error, declaration_errors))?; + let classes: ContractClasses<'py> = contract_classes(py)?.extract()?; + column_tuple(py, &classes, &columns, column_locations) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/headers/thresholds.rs b/crates/sqlbuild-python-front/src/model_config/main/headers/thresholds.rs index 74ba4ef14c..8524b7a440 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/headers/thresholds.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/headers/thresholds.rs @@ -1,19 +1,25 @@ //! `MeasurementThresholds` of `parsed`, or `None`. -use crate::model_config::_helpers::headers::header_metadata; +use crate::model_config::_helpers::headers::header_metadata::contract_object; +use crate::model_config::_helpers::headers::header_metadata::threshold_bound; use crate::model_config::models::ContractClasses; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use sqlbuild_model_config::header_metadata::models::ParsedThresholds; /// `MeasurementThresholds` of `parsed`, or `None`. -#[inline] pub fn thresholds<'py>( py: Python<'py>, classes: &ContractClasses<'py>, parsed: Option<&ParsedThresholds>, ) -> PyResult> { - header_metadata::thresholds(py, classes, parsed) + let Some(parsed) = parsed else { + return Ok(py.None().into_bound(py)); + }; + contract_object( + &classes.measurement_thresholds, + [ + threshold_bound(py, classes, parsed.warn)?, + threshold_bound(py, classes, parsed.error)?, + ], + ) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/column_migration_error.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/column_migration_error.rs index 583bffaebc..81c6ac15f5 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/column_migration_error.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/column_migration_error.rs @@ -1,18 +1,35 @@ //! The first column-rename problem for a model with these config values, if any. -use crate::model_config::_helpers::model_nodes::column_migrations; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use crate::model_config::_helpers::model_nodes::column_migrations::MATERIALIZED_KEY; +use crate::model_config::_helpers::model_nodes::column_migrations::MIGRATE_FROM_KEY; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; use sqlbuild_model_config::errors::ConfigError; +use sqlbuild_model_config::model_validation::main::validate_column_migrations::validate_column_migrations; use sqlbuild_model_config::model_validation::models::ColumnMigration; /// The first column-rename problem for a model with these config values, if any. -#[inline] pub fn column_migration_error( model_name: &str, values: &Bound<'_, PyDict>, columns: &[ColumnMigration], ) -> PyResult> { - column_migrations::column_migration_error(model_name, values, columns) + if columns.iter().all(|column| column.migrate_from.is_none()) { + return Ok(None); + } + let materialized: Option = match values.get_item(MATERIALIZED_KEY)? { + None => None, + Some(value) if value.is_none() => None, + Some(value) => match value.downcast::() { + Ok(text) => Some(text.to_str()?.to_owned()), + Err(_) => return Ok(Some(ConfigError::config_value_type(MATERIALIZED_KEY))), + }, + }; + let model_migrates: bool = values + .get_item(MIGRATE_FROM_KEY)? + .is_some_and(|value| !value.is_none()); + Ok( + validate_column_migrations(model_name, materialized.as_deref(), model_migrates, columns) + .err(), + ) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/contract_classes.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/contract_classes.rs index 47eaec549e..46b6ce4e6c 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/contract_classes.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/contract_classes.rs @@ -1,13 +1,13 @@ //! The contract classes native header parsing builds columns and audits as. -use crate::model_config::_helpers::model_nodes::contract_classes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::_helpers::model_nodes::contract_classes::CONTRACT_CLASS_TABLE; +use crate::model_config::_helpers::model_nodes::contract_classes::built_contract_classes; +use pyo3::prelude::{Bound, PyResult, Python}; use pyo3::types::PyDict; /// The contract classes native header parsing builds columns and audits as. -#[inline] pub fn contract_classes(py: Python<'_>) -> PyResult> { - contract_classes::contract_classes(py) + CONTRACT_CLASS_TABLE + .get_or_try_init(py, || built_contract_classes(py).map(Bound::unbind)) + .map(|classes| classes.bind(py).clone()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/externally_described_models.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/externally_described_models.rs index 699db0625e..17504a2ff0 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/externally_described_models.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/externally_described_models.rs @@ -1,9 +1,7 @@ //! The file path of each model whose layered config or model schema gives it a description. use crate::model_config::_helpers::model_nodes::described_models; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; /// The file path of each model whose layered config or model schema gives it a description. diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/raised_config_error.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/raised_config_error.rs index 62a6e2593b..76fe232d6d 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/raised_config_error.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/raised_config_error.rs @@ -1,12 +1,20 @@ //! The Python exception a native config error raises as. -use crate::model_config::_helpers::model_nodes::model_config_builder; +use crate::model_config::_helpers::config::config_errors::config_exception; +use crate::model_config::errors::NativeConfigError; +use crate::model_config::main::config::native_config_error::native_config_error; use pyo3::PyErr; -use pyo3::prelude::Python; +use pyo3::prelude::{Py, PyResult, Python}; use sqlbuild_model_config::errors::ConfigError; /// The Python exception a native config error raises as. -#[inline] pub fn raised_config_error(py: Python<'_>, error: ConfigError) -> PyErr { - model_config_builder::raised_config_error(py, error) + let raised = || -> PyResult { + let error: Py = native_config_error(py, error)?; + Ok(PyErr::from_value(config_exception( + error.bind(py).as_any(), + None, + )?)) + }; + raised().unwrap_or_else(|error| error) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_config_builder.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_config_builder.rs index c191e8b135..c36feeb817 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_config_builder.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_config_builder.rs @@ -1,12 +1,10 @@ //! Register the model config builder functions and classes. -use crate::model_config::_helpers::model_nodes::model_config_builder; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::model_config::models::NativeModelConfigBuilder; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the model config builder functions and classes. -#[inline] pub fn register_model_config_builder(module: &Bound<'_, PyModule>) -> PyResult<()> { - model_config_builder::register(module) + module.add_class::()?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_validator.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_validator.rs index bd5ccb345e..eb1686b972 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_validator.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_model_validator.rs @@ -1,12 +1,10 @@ //! Register the model validator functions and classes. -use crate::model_config::_helpers::model_nodes::model_validator; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::model_config::models::NativeModelValidator; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the model validator functions and classes. -#[inline] pub fn register_model_validator(module: &Bound<'_, PyModule>) -> PyResult<()> { - model_validator::register(module) + module.add_class::()?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_unicode_environment.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_unicode_environment.rs index 6399b67927..c1af74c7ab 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_unicode_environment.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/register_unicode_environment.rs @@ -1,12 +1,9 @@ //! Register the unicode environment functions and classes. -use crate::model_config::_helpers::model_nodes::unicode_environment; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::model_config::_helpers::model_nodes::unicode_environment::UnicodeEnvironment; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the unicode environment functions and classes. -#[inline] pub fn register_unicode_environment(module: &Bound<'_, PyModule>) -> PyResult<()> { - unicode_environment::register(module) + module.add_class::() } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/unicode_environment.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/unicode_environment.rs index 59c24504a6..d15d1bc440 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/unicode_environment.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/unicode_environment.rs @@ -1,13 +1,9 @@ //! A new view of the process environment as compile text reads it. -use crate::model_config::_helpers::model_nodes::unicode_environment; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::_helpers::model_nodes::unicode_environment::UnicodeEnvironment; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; /// A new view of the process environment as compile text reads it. -#[inline] pub fn unicode_environment(py: Python<'_>) -> PyResult> { - unicode_environment::unicode_environment(py) + Ok(Bound::new(py, UnicodeEnvironment)?.into_any()) } diff --git a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/yaml_entry_classes.rs b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/yaml_entry_classes.rs index bc68a1655a..f0e9b701ee 100644 --- a/crates/sqlbuild-python-front/src/model_config/main/model_nodes/yaml_entry_classes.rs +++ b/crates/sqlbuild-python-front/src/model_config/main/model_nodes/yaml_entry_classes.rs @@ -1,13 +1,13 @@ //! The classes parsed seed and source declarations become. -use crate::model_config::_helpers::model_nodes::contract_classes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::model_config::_helpers::model_nodes::contract_classes::YAML_ENTRY_CLASS_TABLE; +use crate::model_config::_helpers::model_nodes::contract_classes::built_yaml_entry_classes; +use pyo3::prelude::{Bound, PyResult, Python}; use pyo3::types::PyDict; /// The classes parsed seed and source declarations become. -#[inline] pub fn yaml_entry_classes(py: Python<'_>) -> PyResult> { - contract_classes::yaml_entry_classes(py) + YAML_ENTRY_CLASS_TABLE + .get_or_try_init(py, || built_yaml_entry_classes(py).map(Bound::unbind)) + .map(|classes| classes.bind(py).clone()) } diff --git a/crates/sqlbuild-python-front/src/model_config/models.rs b/crates/sqlbuild-python-front/src/model_config/models.rs index 2b2d80cc80..c1b99e21ea 100644 --- a/crates/sqlbuild-python-front/src/model_config/models.rs +++ b/crates/sqlbuild-python-front/src/model_config/models.rs @@ -6,12 +6,9 @@ use crate::authored_values::main::project_var_text::project_var_text; use crate::discovery::main::project::each::each; use crate::discovery::models::NativeModelFile; use crate::model_config::_helpers::config::config_classes::built_config; -use crate::model_config::_helpers::config::config_errors::native_config_error; use crate::model_config::_helpers::config::config_nodes::PyConfigEntries; use crate::model_config::_helpers::config::config_nodes::config_dict; use crate::model_config::_helpers::config::config_nodes::config_entries; -use crate::model_config::_helpers::config::config_nodes::config_node; -use crate::model_config::_helpers::config::config_nodes::config_object; use crate::model_config::_helpers::config::config_templates::CONTEXT_READ; use crate::model_config::_helpers::config::config_templates::EFFECTIVE_VARS_LABEL; use crate::model_config::_helpers::config::config_templates::ENVIRONMENT_READ; @@ -19,7 +16,6 @@ use crate::model_config::_helpers::config::config_templates::EffectiveVars; use crate::model_config::_helpers::headers::header_audits::authored_audit; use crate::model_config::_helpers::headers::header_audits::config_json_value; use crate::model_config::_helpers::headers::header_audits::retain_audit; -use crate::model_config::_helpers::headers::header_families::family_specs; use crate::model_config::_helpers::headers::header_metadata::ColumnFields; use crate::model_config::_helpers::headers::header_metadata::SCHEMA_COLUMN_FIELDS; use crate::model_config::_helpers::headers::header_metadata::audit_instance; @@ -28,9 +24,7 @@ use crate::model_config::_helpers::headers::header_metadata::contract_object; use crate::model_config::_helpers::headers::header_metadata::native_location; use crate::model_config::_helpers::headers::header_metadata::native_value; use crate::model_config::_helpers::headers::header_metadata::native_values; -use crate::model_config::_helpers::headers::header_metadata::thresholds; use crate::model_config::_helpers::model_nodes::authored_nodes::entry_nodes; -use crate::model_config::_helpers::model_nodes::contract_classes::contract_classes; use crate::model_config::_helpers::model_nodes::model_config_builder::CONTEXT_KEYS; use crate::model_config::_helpers::model_nodes::model_config_builder::HeaderBuildInputs; use crate::model_config::_helpers::model_nodes::model_config_builder::PythonConfigHost; @@ -46,6 +40,12 @@ use crate::model_config::_helpers::model_nodes::model_validator::discovered_name use crate::model_config::_helpers::model_nodes::model_validator::stop_object; use crate::model_config::_helpers::model_nodes::project_defaults::project_defaults_mapping; use crate::model_config::errors::NativeConfigError; +use crate::model_config::main::config::config_node::config_node; +use crate::model_config::main::config::config_object::config_object; +use crate::model_config::main::config::native_config_error::native_config_error; +use crate::model_config::main::headers::family_specs::family_specs; +use crate::model_config::main::headers::thresholds::thresholds; +use crate::model_config::main::model_nodes::contract_classes::contract_classes; use crate::model_config::types::BuiltConfigValues; use crate::model_config::types::HeaderValue; use crate::model_config::types::PyConfigNode; diff --git a/crates/sqlbuild-python-front/src/model_config/types.rs b/crates/sqlbuild-python-front/src/model_config/types.rs index bf2205a325..26a64f4fcc 100644 --- a/crates/sqlbuild-python-front/src/model_config/types.rs +++ b/crates/sqlbuild-python-front/src/model_config/types.rs @@ -1,6 +1,6 @@ //! Public traits and type aliases of model config. -use crate::model_config::_helpers::config::config_nodes::config_node; +use crate::model_config::main::config::config_node::config_node; use crate::model_config::models::HostHandle; use crate::model_config::models::TypedConfigValues; use pyo3::prelude::Bound; diff --git a/crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs b/crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs index 2a2ecb2147..f6947339b1 100644 --- a/crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs @@ -1,16 +1,15 @@ //! Pruning of the audit results state table through the adapter. -use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::native_host::main::state_table_name::state_table_name; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use sqlbuild_core::constants::AUDIT_RESULTS_TABLE_NAME; use sqlbuild_state::audit_results::main::prune_audit_results_sql::prune_audit_results_sql; -use crate::native_host::_helpers::consumption_state::state_table_name; - /// SQL deleting audit results beyond the newest `retain_versions` per audit binding. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name, retain_versions))] -fn build_audit_results_prune_sql( +pub(crate) fn build_audit_results_prune_sql( database: &Bound<'_, PyAny>, schema: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, @@ -24,7 +23,3 @@ fn build_audit_results_prune_sql( .map(|table| prune_audit_results_sql(&table, retain_versions)) .unwrap_or_default()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_audit_results_prune_sql, module)?) -} diff --git a/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs b/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs index 59e36ee016..4414633a9f 100644 --- a/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs @@ -1,16 +1,22 @@ //! The consumption table through the adapter, and the connection and callbacks a recording build holds. use crate::native_host::constants::ROWS_PER_STATEMENT; +use crate::native_host::main::execute::execute; +use crate::native_host::main::fixed_state_table_sql::fixed_state_table_sql; +use crate::native_host::main::integer_literal::integer_literal; +use crate::native_host::main::literal::literal; +use crate::native_host::main::optional_text::optional_text; +use crate::native_host::main::state_table_name::state_table_name; use crate::native_host::models::Location; use chrono::{DateTime, NaiveDateTime, Utc}; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyFrozenSet, PyInt, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyInt, PyString}; use sqlbuild_core::constants::CONSUMPTION_TABLE_NAME; use sqlbuild_state::consumption::constants::{ - CONSUMPTION_COLUMN_TYPES, INTEGER_STATE_VALUE_TYPE, NULLABLE_COLUMNS, RECENT_EXECUTIONS, + CONSUMPTION_COLUMN_TYPES, NULLABLE_COLUMNS, RECENT_EXECUTIONS, }; use sqlbuild_state::consumption::main::executions_by_id_sql::executions_by_id_sql; use sqlbuild_state::consumption::main::prune_executions_sql::prune_executions_sql; @@ -18,10 +24,8 @@ use sqlbuild_state::consumption::main::recent_executions_sql::recent_executions_ use sqlbuild_state::consumption::models::{ ConsumedInput, ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey, }; -use sqlbuild_state::freshness::constants::STRING_STATE_VALUE_TYPE; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; -use crate::state_rows::main::tables::state_table_create_sql::state_table_create_sql; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::contracts::main::render_state_sql_literal::render_state_sql_literal; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -37,27 +41,6 @@ pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() } -/// The adapter's qualified name of a state table in `(database, schema)`, if it has one. -pub(crate) fn state_table_name( - render_qualified_name: &Bound<'_, PyAny>, - (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - table: &str, -) -> PyResult> { - let py: Python<'_> = render_qualified_name.py(); - let name: Bound<'_, PyAny> = call_with( - render_qualified_name, - &[ - ("database", database.clone()), - ("schema", schema.clone()), - ("name", text(py, table)), - ], - )?; - if name.is_none() { - return Ok(None); - } - Ok(Some(name.extract()?)) -} - fn table_of(adapter: &Bound<'_, PyAny>, location: &Location) -> PyResult> { let py: Python<'_> = adapter.py(); let database: Bound<'_, PyAny> = location @@ -74,7 +57,7 @@ fn table_of(adapter: &Bound<'_, PyAny>, location: &Location) -> PyResult, schema: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, @@ -91,43 +74,10 @@ fn build_consumption_create_table_sql( ) } -/// Create-if-missing DDL for a state table with fixed columns, typed by the adapter. -pub(crate) fn fixed_state_table_sql( - (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - (render_qualified_name, render_framework_type): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - (table_name, column_types, nullable): (&str, &[(&str, &str)], &[&str]), -) -> PyResult { - let py: Python<'_> = database.py(); - let Some(table) = state_table_name(render_qualified_name, (database, schema), table_name)? - else { - return Ok(String::new()); - }; - let columns: Bound<'_, PyAny> = - PyTuple::new(py, column_types.iter().map(|(column, _)| *column))?.into_any(); - let types: Bound<'_, PyDict> = PyDict::new(py); - for (column, declared) in column_types { - types.set_item(column, declared)?; - } - let required: Vec<&str> = column_types - .iter() - .map(|(column, _)| *column) - .filter(|column| !nullable.contains(column)) - .collect(); - state_table_create_sql( - &table, - ( - &columns, - &types.into_any(), - &PyFrozenSet::new(py, &required)?.into_any(), - ), - render_framework_type, - ) -} - /// SQL deleting whole consumption executions beyond the newest `retain_versions` per relation. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name, retain_versions))] -fn build_consumption_prune_sql( +pub(crate) fn build_consumption_prune_sql( database: &Bound<'_, PyAny>, schema: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, @@ -142,20 +92,6 @@ fn build_consumption_prune_sql( .unwrap_or_default()) } -pub(crate) fn execute<'py>( - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, - sql: &str, -) -> PyResult> { - call_with( - &native_field(adapter, "execute")?, - &[ - ("connection", connection.clone()), - ("sql", text(adapter.py(), sql)), - ], - ) -} - /// Create the schema, table and indexes in `location`; `started` rows precede schema preparation. pub(crate) fn ensure_table( adapter: &Bound<'_, PyAny>, @@ -248,13 +184,6 @@ pub(crate) fn read_executions( Ok(rows) } -pub(crate) fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { - if value.is_none() { - return Ok(None); - } - Ok(Some(value.str()?.extract()?)) -} - fn required_text(value: &Bound<'_, PyAny>) -> PyResult { Ok(optional_text(value)?.unwrap_or_default()) } @@ -323,7 +252,7 @@ fn read_row(row: &Bound<'_, PyAny>) -> PyResult { } /// A nullable state value as a SQL literal of `declared_type`. -fn state_literal( +pub(crate) fn state_literal( py: Python<'_>, value: Option>, declared_type: &str, @@ -332,22 +261,6 @@ fn state_literal( render_state_sql_literal(py, &value, declared_type, None) } -pub(crate) fn literal(py: Python<'_>, value: Option<&str>) -> PyResult { - state_literal( - py, - value.map(|value| text(py, value)), - STRING_STATE_VALUE_TYPE, - ) -} - -pub(crate) fn integer_literal(py: Python<'_>, value: Option) -> PyResult { - state_literal( - py, - value.map(|value| PyInt::new(py, value).into_any()), - INTEGER_STATE_VALUE_TYPE, - ) -} - fn row_cells(py: Python<'_>, row: &ConsumptionAppend, recorded_at: &str) -> PyResult> { let input = row.input.as_ref(); Ok(vec![ @@ -398,21 +311,3 @@ pub(crate) fn append( } Ok(()) } - -/// The recorder's `on_node_complete` and `before_node_write` methods, as build callbacks. -pub(crate) fn recorder_callbacks<'py>( - recorder: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - Ok(( - recorder.getattr("on_node_complete")?, - recorder.getattr("before_node_write")?, - )) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - build_consumption_create_table_sql, - module - )?)?; - module.add_function(wrap_pyfunction!(build_consumption_prune_sql, module)?) -} diff --git a/crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs b/crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs index bb8276aadc..72254144d4 100644 --- a/crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs @@ -1,29 +1,19 @@ //! DDL and pruning of the display-only observations table through the adapter. -use crate::native_host::models::ObservationAppend; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::native_host::main::fixed_state_table_sql::fixed_state_table_sql; +use crate::native_host::main::state_table_name::state_table_name; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use sqlbuild_core::constants::OBSERVATIONS_TABLE_NAME; use sqlbuild_state::observations::constants::{ NULLABLE_OBSERVATION_COLUMNS, OBSERVATION_COLUMN_TYPES, }; use sqlbuild_state::observations::main::prune_observations_sql::prune_observations_sql; -use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; -use uuid::Uuid; - -use crate::native_host::_helpers::consumption_state::execute; -use crate::native_host::_helpers::consumption_state::fixed_state_table_sql; -use crate::native_host::_helpers::consumption_state::literal; -use crate::native_host::_helpers::consumption_state::state_table_name; -use crate::native_host::constants::ROWS_PER_STATEMENT; -use crate::native_host::models::Location; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Create-if-missing DDL for the observations table, with adapter-rendered column types. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name, render_framework_type))] -fn build_observations_create_table_sql( +pub(crate) fn build_observations_create_table_sql( database: &Bound<'_, PyAny>, schema: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, @@ -43,7 +33,7 @@ fn build_observations_create_table_sql( /// SQL deleting observations beyond the newest `retain_versions` per kind and node. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name, retain_versions))] -fn build_observations_prune_sql( +pub(crate) fn build_observations_prune_sql( database: &Bound<'_, PyAny>, schema: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, @@ -57,71 +47,3 @@ fn build_observations_prune_sql( .map(|table| prune_observations_sql(&table, retain_versions)) .unwrap_or_default()) } - -/// Create the schema and observations table in `location` if missing, then append `rows`. -pub(crate) fn append_observations( - adapter: &Bound<'_, PyAny>, - connection: &Bound<'_, PyAny>, - location: &Location, - rows: &[ObservationAppend], -) -> PyResult<()> { - let py: Python<'_> = adapter.py(); - let keywords = location.keywords(py); - let Some(table) = state_table_name( - &native_field(adapter, "render_qualified_name")?, - (&keywords[0].1, &keywords[1].1), - OBSERVATIONS_TABLE_NAME, - )? - else { - return Ok(()); - }; - for statement in - call_with(&native_field(adapter, "render_create_schema")?, &keywords)?.try_iter()? - { - execute(adapter, connection, &statement?.extract::()?)?; - } - let create: String = call_with( - &native_field(adapter, "render_create_observations_table_sql")?, - &keywords, - )? - .extract()?; - if !create.is_empty() { - execute(adapter, connection, &create)?; - } - let observed_at: String = - call_with(&native_field(adapter, "render_current_utc_timestamp")?, &[])?.extract()?; - let columns: Vec<&str> = OBSERVATION_COLUMN_TYPES - .iter() - .map(|(column, _)| *column) - .collect(); - for chunk in rows.chunks(ROWS_PER_STATEMENT) { - let cells: Vec> = chunk - .iter() - .map(|row| { - Ok(vec![ - literal(py, Some(&Uuid::now_v7().to_string()))?, - literal(py, Some(row.kind))?, - literal(py, Some(&row.node))?, - literal(py, row.cursor_type.as_deref())?, - literal(py, Some(&row.value))?, - literal(py, Some(&row.run_id))?, - observed_at.clone(), - ]) - }) - .collect::>()?; - execute( - adapter, - connection, - &values_insert_sql(&table, &columns, &cells), - )?; - } - Ok(()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - build_observations_create_table_sql, - module - )?)?; - module.add_function(wrap_pyfunction!(build_observations_prune_sql, module)?) -} diff --git a/crates/sqlbuild-python-front/src/native_host/main/append_observations.rs b/crates/sqlbuild-python-front/src/native_host/main/append_observations.rs index 5d63443aa6..8fe6343d54 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/append_observations.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/append_observations.rs @@ -1,19 +1,75 @@ //! Create the schema and observations table in `location` if missing, then append `rows`. -use crate::native_host::_helpers::observations_state; +use crate::native_host::constants::ROWS_PER_STATEMENT; +use crate::native_host::main::execute::execute; +use crate::native_host::main::literal::literal; +use crate::native_host::main::state_table_name::state_table_name; use crate::native_host::models::Location; use crate::native_host::models::ObservationAppend; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_core::constants::OBSERVATIONS_TABLE_NAME; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_state::observations::constants::OBSERVATION_COLUMN_TYPES; +use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; +use uuid::Uuid; /// Create the schema and observations table in `location` if missing, then append `rows`. -#[inline] pub fn append_observations( adapter: &Bound<'_, PyAny>, connection: &Bound<'_, PyAny>, location: &Location, rows: &[ObservationAppend], ) -> PyResult<()> { - observations_state::append_observations(adapter, connection, location, rows) + let py: Python<'_> = adapter.py(); + let keywords = location.keywords(py); + let Some(table) = state_table_name( + &native_field(adapter, "render_qualified_name")?, + (&keywords[0].1, &keywords[1].1), + OBSERVATIONS_TABLE_NAME, + )? + else { + return Ok(()); + }; + for statement in + call_with(&native_field(adapter, "render_create_schema")?, &keywords)?.try_iter()? + { + execute(adapter, connection, &statement?.extract::()?)?; + } + let create: String = call_with( + &native_field(adapter, "render_create_observations_table_sql")?, + &keywords, + )? + .extract()?; + if !create.is_empty() { + execute(adapter, connection, &create)?; + } + let observed_at: String = + call_with(&native_field(adapter, "render_current_utc_timestamp")?, &[])?.extract()?; + let columns: Vec<&str> = OBSERVATION_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect(); + for chunk in rows.chunks(ROWS_PER_STATEMENT) { + let cells: Vec> = chunk + .iter() + .map(|row| { + Ok(vec![ + literal(py, Some(&Uuid::now_v7().to_string()))?, + literal(py, Some(row.kind))?, + literal(py, Some(&row.node))?, + literal(py, row.cursor_type.as_deref())?, + literal(py, Some(&row.value))?, + literal(py, Some(&row.run_id))?, + observed_at.clone(), + ]) + }) + .collect::>()?; + execute( + adapter, + connection, + &values_insert_sql(&table, &columns, &cells), + )?; + } + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/execute.rs b/crates/sqlbuild-python-front/src/native_host/main/execute.rs index 1c9e3b6ff0..b0b3f91534 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/execute.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/execute.rs @@ -1,16 +1,21 @@ //! Execute. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::native_host::_helpers::consumption_state::text; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Execute. -#[inline] pub fn execute<'py>( adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, sql: &str, ) -> PyResult> { - consumption_state::execute(adapter, connection, sql) + call_with( + &native_field(adapter, "execute")?, + &[ + ("connection", connection.clone()), + ("sql", text(adapter.py(), sql)), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/fixed_state_table_sql.rs b/crates/sqlbuild-python-front/src/native_host/main/fixed_state_table_sql.rs index 91cfd5cb21..93299a7407 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/fixed_state_table_sql.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/fixed_state_table_sql.rs @@ -1,20 +1,39 @@ //! Create-if-missing DDL for a state table with fixed columns, typed by the adapter. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::native_host::main::state_table_name::state_table_name; +use crate::state_rows::main::tables::state_table_create_sql::state_table_create_sql; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; /// Create-if-missing DDL for a state table with fixed columns, typed by the adapter. -#[inline] pub fn fixed_state_table_sql( (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), (render_qualified_name, render_framework_type): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), (table_name, column_types, nullable): (&str, &[(&str, &str)], &[&str]), ) -> PyResult { - consumption_state::fixed_state_table_sql( - (database, schema), - (render_qualified_name, render_framework_type), - (table_name, column_types, nullable), + let py: Python<'_> = database.py(); + let Some(table) = state_table_name(render_qualified_name, (database, schema), table_name)? + else { + return Ok(String::new()); + }; + let columns: Bound<'_, PyAny> = + PyTuple::new(py, column_types.iter().map(|(column, _)| *column))?.into_any(); + let types: Bound<'_, PyDict> = PyDict::new(py); + for (column, declared) in column_types { + types.set_item(column, declared)?; + } + let required: Vec<&str> = column_types + .iter() + .map(|(column, _)| *column) + .filter(|column| !nullable.contains(column)) + .collect(); + state_table_create_sql( + &table, + ( + &columns, + &types.into_any(), + &PyFrozenSet::new(py, &required)?.into_any(), + ), + render_framework_type, ) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/integer_literal.rs b/crates/sqlbuild-python-front/src/native_host/main/integer_literal.rs index 290ad1dd2a..b20b3d4ebf 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/integer_literal.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/integer_literal.rs @@ -1,11 +1,15 @@ //! Integer literal. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::native_host::_helpers::consumption_state::state_literal; +use pyo3::prelude::{PyResult, Python}; +use pyo3::types::PyInt; +use sqlbuild_state::consumption::constants::INTEGER_STATE_VALUE_TYPE; /// Integer literal. -#[inline] pub fn integer_literal(py: Python<'_>, value: Option) -> PyResult { - consumption_state::integer_literal(py, value) + state_literal( + py, + value.map(|value| PyInt::new(py, value).into_any()), + INTEGER_STATE_VALUE_TYPE, + ) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/literal.rs b/crates/sqlbuild-python-front/src/native_host/main/literal.rs index 5a274ff9fb..c322e768a0 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/literal.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/literal.rs @@ -1,11 +1,15 @@ //! Literal. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::native_host::_helpers::consumption_state::state_literal; +use crate::native_host::_helpers::consumption_state::text; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_state::freshness::constants::STRING_STATE_VALUE_TYPE; /// Literal. -#[inline] pub fn literal(py: Python<'_>, value: Option<&str>) -> PyResult { - consumption_state::literal(py, value) + state_literal( + py, + value.map(|value| text(py, value)), + STRING_STATE_VALUE_TYPE, + ) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/optional_text.rs b/crates/sqlbuild-python-front/src/native_host/main/optional_text.rs index 1babb19549..affa4292b7 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/optional_text.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/optional_text.rs @@ -1,12 +1,11 @@ //! Optional text. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// Optional text. -#[inline] pub fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { - consumption_state::optional_text(value) + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.extract()?)) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/recorder_callbacks.rs b/crates/sqlbuild-python-front/src/native_host/main/recorder_callbacks.rs index 3a3544c506..a9e0baa682 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/recorder_callbacks.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/recorder_callbacks.rs @@ -1,14 +1,13 @@ //! The recorder's `on_node_complete` and `before_node_write` methods, as build callbacks. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// The recorder's `on_node_complete` and `before_node_write` methods, as build callbacks. -#[inline] pub fn recorder_callbacks<'py>( recorder: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - consumption_state::recorder_callbacks(recorder) + Ok(( + recorder.getattr("on_node_complete")?, + recorder.getattr("before_node_write")?, + )) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/register_audit_results_state.rs b/crates/sqlbuild-python-front/src/native_host/main/register_audit_results_state.rs index d7cdba3673..1fd72949b3 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/register_audit_results_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/register_audit_results_state.rs @@ -1,12 +1,10 @@ //! Register the audit results state functions and classes. -use crate::native_host::_helpers::audit_results_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::native_host::_helpers::audit_results_state::build_audit_results_prune_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the audit results state functions and classes. -#[inline] pub fn register_audit_results_state(module: &Bound<'_, PyModule>) -> PyResult<()> { - audit_results_state::register(module) + module.add_function(wrap_pyfunction!(build_audit_results_prune_sql, module)?) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/register_consumption_state.rs b/crates/sqlbuild-python-front/src/native_host/main/register_consumption_state.rs index 84da4f5782..6fc37e3d1f 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/register_consumption_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/register_consumption_state.rs @@ -1,12 +1,15 @@ //! Register the consumption state functions and classes. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::native_host::_helpers::consumption_state::build_consumption_create_table_sql; +use crate::native_host::_helpers::consumption_state::build_consumption_prune_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the consumption state functions and classes. -#[inline] pub fn register_consumption_state(module: &Bound<'_, PyModule>) -> PyResult<()> { - consumption_state::register(module) + module.add_function(wrap_pyfunction!( + build_consumption_create_table_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(build_consumption_prune_sql, module)?) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs b/crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs index 97ab0d3395..d94dbf8327 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/register_observations_state.rs @@ -1,12 +1,15 @@ //! Register the observations state functions and classes. -use crate::native_host::_helpers::observations_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::native_host::_helpers::observations_state::build_observations_create_table_sql; +use crate::native_host::_helpers::observations_state::build_observations_prune_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the observations state functions and classes. -#[inline] pub fn register_observations_state(module: &Bound<'_, PyModule>) -> PyResult<()> { - observations_state::register(module) + module.add_function(wrap_pyfunction!( + build_observations_create_table_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(build_observations_prune_sql, module)?) } diff --git a/crates/sqlbuild-python-front/src/native_host/main/state_table_name.rs b/crates/sqlbuild-python-front/src/native_host/main/state_table_name.rs index 5d1c8fd59c..26fb727b81 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/state_table_name.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/state_table_name.rs @@ -1,16 +1,26 @@ //! The adapter's qualified name of a state table in `(database, schema)`, if it has one. -use crate::native_host::_helpers::consumption_state; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::native_host::_helpers::consumption_state::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::boundary::main::call_with::call_with; /// The adapter's qualified name of a state table in `(database, schema)`, if it has one. -#[inline] pub fn state_table_name( render_qualified_name: &Bound<'_, PyAny>, (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), table: &str, ) -> PyResult> { - consumption_state::state_table_name(render_qualified_name, (database, schema), table) + let py: Python<'_> = render_qualified_name.py(); + let name: Bound<'_, PyAny> = call_with( + render_qualified_name, + &[ + ("database", database.clone()), + ("schema", schema.clone()), + ("name", text(py, table)), + ], + )?; + if name.is_none() { + return Ok(None); + } + Ok(Some(name.extract()?)) } diff --git a/crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs b/crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs index 374c433043..81b7873466 100644 --- a/crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs +++ b/crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs @@ -1,9 +1,9 @@ //! Contract adoption's comparison of authored contracts with physical warehouse columns. use pyo3::exceptions::PyException; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::text::main::python_casefold::python_casefold; use sqlbuild_host::entries::models::AdapterHost; @@ -24,7 +24,7 @@ const SOURCE_RESOURCE: &str = "source"; /// Python callables and classes the comparison builds findings with. struct Vocabulary<'py> { finding: Bound<'py, PyAny>, - kinds: Bound<'py, PyAny>, + pub(crate) kinds: Bound<'py, PyAny>, } impl<'py> Vocabulary<'py> { @@ -435,7 +435,7 @@ fn compared<'py>( /// `item` compared with its physical columns, or marked missing when there are none. #[pyfunction] #[pyo3(signature = (*, item, physical_columns, adapter))] -fn compare_contract_evidence<'py>( +pub(crate) fn compare_contract_evidence<'py>( py: Python<'py>, item: &Bound<'py, PyAny>, physical_columns: &Bound<'py, PyAny>, @@ -508,9 +508,3 @@ pub(crate) fn compare_contracts<'py>( } PyTuple::new(py, evidence) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(compare_contracts, module)?)?; - module.add_function(wrap_pyfunction!(compare_contract_evidence, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/refactor/_helpers/contract_edits.rs b/crates/sqlbuild-python-front/src/refactor/_helpers/contract_edits.rs index 02a4a5b0a4..9a2217131f 100644 --- a/crates/sqlbuild-python-front/src/refactor/_helpers/contract_edits.rs +++ b/crates/sqlbuild-python-front/src/refactor/_helpers/contract_edits.rs @@ -1,36 +1,24 @@ //! Contract adoption's MODEL header and YAML source edits, computed natively. use pyo3::exceptions::PyValueError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use sqlbuild_refactor::refactoring::errors::{RefactorError, RefactorErrorKind}; -use sqlbuild_refactor::refactoring::main::edit_model_contract::edit_model_contract; -use sqlbuild_refactor::refactoring::main::edit_source_contract::edit_source_contract; -use sqlbuild_refactor::refactoring::models::{ - ContractEdit, ContractModelColumn, ContractModelRequest, ContractSourceRequest, -}; +use sqlbuild_refactor::refactoring::models::ContractEdit; use sqlbuild_refactor::refactoring::types::PhysicalColumn; -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_base::records::types::RecordId; /// Finding kinds an edit resolves without overwrite, then the ones overwrite also resolves. -const RESOLVED_KINDS: ([&str; 1], [&str; 2]) = ( +pub(crate) const RESOLVED_KINDS: ([&str; 1], [&str; 2]) = ( ["missing_declaration"], ["type_mismatch", "missing_physical_column"], ); /// `(edited contents, ownership conflict, ContractAdoptionError message)`. -type EditRow = (Option, Option, Option); -fn edit_row(result: Result) -> PyResult { +pub(crate) type EditRow = (Option, Option, Option); +pub(crate) fn edit_row(result: Result) -> PyResult { match result { Ok(ContractEdit::Edited(contents)) => Ok((Some(contents), None, None)), Ok(ContractEdit::Conflict(conflict)) => Ok((None, Some(conflict), None)), @@ -42,12 +30,15 @@ fn edit_row(result: Result) -> PyResult { } /// `ContractAdoptionError(message)`. -fn adoption_error(py: Python<'_>, message: String) -> PyErr { +pub(crate) fn adoption_error(py: Python<'_>, message: String) -> PyErr { new_plain_error(py, ErrorId::ContractAdoption, &message) } /// `(edited contents, ownership conflict)`, raising the edit's adoption failure. -fn edit_outcome(py: Python<'_>, row: EditRow) -> PyResult<(Option, Option)> { +pub(crate) fn edit_outcome( + py: Python<'_>, + row: EditRow, +) -> PyResult<(Option, Option)> { let (contents, conflict, failure) = row; match failure { Some(failure) => Err(adoption_error(py, failure)), @@ -55,7 +46,7 @@ fn edit_outcome(py: Python<'_>, row: EditRow) -> PyResult<(Option, Optio } } -fn physical_rows(columns: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn physical_rows(columns: &Bound<'_, PyAny>) -> PyResult> { let mut rows: Vec = Vec::new(); for column in columns.try_iter()? { let column: Bound<'_, PyAny> = column?; @@ -66,130 +57,3 @@ fn physical_rows(columns: &Bound<'_, PyAny>) -> PyResult> { } Ok(rows) } - -/// The model's MODEL header with physical columns adopted, or the ownership conflict. -#[pyfunction] -#[pyo3(signature = (*, model, physical_columns, overwrite))] -pub(crate) fn edit_contract_model( - py: Python<'_>, - model: &Bound<'_, PyAny>, - physical_columns: &Bound<'_, PyAny>, - overwrite: bool, -) -> PyResult<(Option, Option)> { - let schema: Bound<'_, PyAny> = native_field(model, "schema_entry")?; - let mut columns: Vec = Vec::new(); - let mut dynamic_patterns: Vec> = Vec::new(); - let shared_schema: bool = - !schema.is_none() && !native_field(&schema, "model_schema")?.is_none(); - if !schema.is_none() { - for column in native_field(&schema, "columns")?.try_iter()? { - let column: Bound<'_, PyAny> = column?; - let location: Bound<'_, PyAny> = native_field(&column, "location")?; - let location_path: Option = if location.is_none() { - None - } else { - Some(native_field(&location, "path")?.str()?.extract()?) - }; - columns.push(ContractModelColumn { - name: native_field(&column, "name")?.extract()?, - declared_type: native_field(&column, "type")?.extract()?, - location_path, - }); - } - for family in native_field(&schema, "dynamic_columns")?.try_iter()? { - dynamic_patterns.push(native_field(&family?, "name_pattern")?.extract()?); - } - } - let request = ContractModelRequest { - contents: native_field(model, "authored_sql")?.extract()?, - relative_path: native_field(model, "relative_path")?.str()?.extract()?, - shared_schema, - columns, - dynamic_patterns, - physical_columns: physical_rows(physical_columns)?, - overwrite, - }; - let row: EditRow = compiler_guard(|| edit_row(edit_model_contract(&request)))?; - edit_outcome(py, row) -} - -/// The source's YAML declaration with physical columns adopted, or the ownership conflict. -#[pyfunction] -#[pyo3(signature = (*, source, physical_columns, overwrite, contents = None))] -pub(crate) fn edit_contract_source( - source: &Bound<'_, PyAny>, - physical_columns: &Bound<'_, PyAny>, - overwrite: bool, - contents: Option, -) -> PyResult<(Option, Option)> { - let discovered_contents: String = - native_field(&native_field(source, "source_file")?, "contents")?.extract()?; - let mut columns: Vec<(String, Option)> = Vec::new(); - for column in native_field(&native_field(source, "source_entry")?, "columns")?.try_iter()? { - let column: Bound<'_, PyAny> = column?; - columns.push(( - native_field(&column, "name")?.extract()?, - native_field(&column, "type")?.extract()?, - )); - } - let request = ContractSourceRequest { - contents: contents.unwrap_or_else(|| discovered_contents.clone()), - discovered_contents, - name: native_field(source, "name")?.extract()?, - columns, - physical_columns: physical_rows(physical_columns)?, - overwrite, - }; - let row: EditRow = compiler_guard(|| edit_row(Ok(edit_source_contract(&request))))?; - edit_outcome(source.py(), row) -} - -/// The findings an edit leaves, plus an ownership-conflict finding when the edit was refused. -#[pyfunction] -#[pyo3(signature = (*, findings, overwrite, conflict))] -pub(crate) fn remaining_contract_findings<'py>( - py: Python<'py>, - findings: &Bound<'py, PyAny>, - overwrite: bool, - conflict: Option, -) -> PyResult> { - let findings: Vec> = findings.try_iter()?.collect::>()?; - if let Some(conflict) = conflict { - let Some(first) = findings.first() else { - return Err(adoption_error( - py, - "contract ownership conflict requires resource evidence".to_owned(), - )); - }; - let keywords: Bound<'py, PyDict> = PyDict::new(py); - keywords.set_item("resource_type", native_field(first, "resource_type")?)?; - keywords.set_item("resource_name", native_field(first, "resource_name")?)?; - keywords.set_item( - "kind", - enum_member(py, RecordId::ContractFindingKind, "OWNERSHIP_CONFLICT")?, - )?; - keywords.set_item("message", conflict)?; - let ownership: Bound<'py, PyAny> = - new_record(py, RecordId::ContractFinding, (), Some(&keywords))?; - let mut remaining: Vec> = findings.clone(); - remaining.push(ownership); - return PyTuple::new(py, remaining); - } - let mut remaining: Vec> = Vec::new(); - for finding in findings { - let kind: String = native_field(&finding, "kind")?.str()?.extract()?; - let resolved: bool = RESOLVED_KINDS.0.contains(&kind.as_str()) - || (overwrite && RESOLVED_KINDS.1.contains(&kind.as_str())); - if !resolved { - remaining.push(finding); - } - } - PyTuple::new(py, remaining) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(edit_contract_model, module)?)?; - module.add_function(wrap_pyfunction!(edit_contract_source, module)?)?; - module.add_function(wrap_pyfunction!(remaining_contract_findings, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs b/crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs index a4df31dcdc..bfbd057003 100644 --- a/crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs +++ b/crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs @@ -2,9 +2,9 @@ use std::path::PathBuf; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_pysource::string_literals::main::string_literal_locations::string_literal_locations; use sqlbuild_pysource::string_literals::models::StringQuery; @@ -100,8 +100,3 @@ pub(crate) fn find_python_string_locations<'py>( } PyTuple::new(py, locations) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(find_python_string_locations, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs b/crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs index bdee9fe23e..e2c2fb5d6d 100644 --- a/crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs +++ b/crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs @@ -7,7 +7,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; use sqlbuild_refactor::refactoring::models::{ AuthoredFile, DeclarationPlacement, DependencyFacts, DestinationFacts, DiscoveredFile, ExpansionFacts, ExpansionSpan, LocationFacts, ManualLocation, ModelFacts, PlacedDeclaration, - RefactorFacts, RefactorOperation, RefactorRequest, RelocatedDeclaration, ResourceColumnsFacts, + RefactorOperation, RefactorRequest, RelocatedDeclaration, ResourceColumnsFacts, SchemaColumnFacts, SqlFileRole, UnitTestFacts, }; use sqlbuild_scopes::scope_index::models::{ResourceIdentity, ResourceKind}; @@ -17,11 +17,10 @@ use crate::refactor::_helpers::python_strings::find_python_string_locations; use crate::scopes::main::checks::relocated_declarations::relocated_declarations; use crate::scopes::main::index::build_scope_lookup::build_scope_lookup; use crate::scopes::main::reports::scope_move_preview::scope_move_preview; -use crate::scopes::main::reports::scope_rebound_usages::scope_rebound_usages; use sqlbuild_python_base::records::main::fields::field::field as native_field; const SQL_SUFFIX: &str = ".sql"; -const GENERIC_DIALECT: &str = "generic"; +pub(crate) const GENERIC_DIALECT: &str = "generic"; const MATERIALIZED_KEY: &str = "materialized"; const MIGRATE_FROM_KEY: &str = "migrate_from"; const MODEL_KIND_PREFIX: &str = "model:"; @@ -30,7 +29,7 @@ fn posix(path: &Bound<'_, PyAny>) -> PyResult { path.call_method0("as_posix")?.extract() } -fn discovered_files(files: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn discovered_files(files: &Bound<'_, PyAny>) -> PyResult> { let mut discovered: Vec = Vec::new(); for item in files.try_iter()? { let item: Bound<'_, PyAny> = item?; @@ -49,7 +48,7 @@ fn column_names(columns: &Bound<'_, PyAny>) -> PyResult> { .collect() } -fn resource_columns( +pub(crate) fn resource_columns( resources: &Bound<'_, PyAny>, entry: &str, ) -> PyResult> { @@ -113,7 +112,10 @@ fn schema_columns(entry: &Bound<'_, PyAny>) -> PyResult, expansion: Option<&Bound<'_, PyAny>>) -> PyResult { +pub(crate) fn model( + model: &Bound<'_, PyAny>, + expansion: Option<&Bound<'_, PyAny>>, +) -> PyResult { let py: Python<'_> = model.py(); let values: Bound<'_, PyDict> = PyDict::new(py); values.call_method1( @@ -158,38 +160,6 @@ fn model(model: &Bound<'_, PyAny>, expansion: Option<&Bound<'_, PyAny>>) -> PyRe }) } -/// Every compiled model's refactoring facts, or none without a project. -pub(crate) fn model_facts( - refactor_project: &Bound<'_, PyAny>, -) -> PyResult>> { - if refactor_project.is_none() { - return Ok(None); - } - let project: Bound<'_, PyAny> = - native_field(&native_field(refactor_project, "graph")?, "project")?; - let project_dir: Bound<'_, PyAny> = native_field(refactor_project, "project_dir")?; - let mut expansions: HashMap> = HashMap::new(); - let items: Vec> = native_field(&project, "sql_expansions")? - .call_method0("items")? - .try_iter()? - .collect::>()?; - for item in items.iter().rev() { - let (path, expansion): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item.extract()?; - expansions.insert(path.call_method0("resolve")?.str()?.extract()?, expansion); - } - let mut facts: Vec = Vec::new(); - for compiled in native_field(&project, "models")?.try_iter()? { - let compiled: Bound<'_, PyAny> = compiled?; - let resolved: String = project_dir - .div(native_field(&compiled, "relative_path")?)? - .call_method0("resolve")? - .str()? - .extract()?; - facts.push(model(&compiled, expansions.get(&resolved))?); - } - Ok(Some(facts)) -} - fn block_bodies(blocks: &Bound<'_, PyAny>) -> PyResult> { blocks .try_iter()? @@ -205,7 +175,7 @@ struct FileGroup { blocks: bool, } -fn authored_files(discovered: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn authored_files(discovered: &Bound<'_, PyAny>) -> PyResult> { let groups: [FileGroup; 6] = [ FileGroup { role: SqlFileRole::Test, @@ -270,7 +240,7 @@ fn authored_files(discovered: &Bound<'_, PyAny>) -> PyResult> Ok(files) } -fn python_locations( +pub(crate) fn python_locations( project: &Bound<'_, PyAny>, request: &RefactorRequest, ) -> PyResult> { @@ -308,7 +278,7 @@ fn python_locations( } /// Whether `request` moves a model without renaming it: a folder, or a file of the same name. -fn move_keeps_name(request: &RefactorRequest) -> bool { +pub(crate) fn move_keeps_name(request: &RefactorRequest) -> bool { if request.operation != RefactorOperation::MoveModel { return false; } @@ -319,44 +289,8 @@ fn move_keeps_name(request: &RefactorRequest) -> bool { } } -/// The facts one refactoring plan of `requests` reads. -pub(crate) fn refactor_facts( - project: &Bound<'_, PyAny>, - requests: &[RefactorRequest], -) -> PyResult { - let mut locations: Vec = Vec::new(); - for request in requests { - if !move_keeps_name(request) { - locations.extend(python_locations(project, request)?); - } - } - let compiled: Bound<'_, PyAny> = native_field(&native_field(project, "graph")?, "project")?; - let discovered: Bound<'_, PyAny> = native_field(project, "discovered")?; - let dialect: Option = native_field(&compiled, "sql_analysis_dialect")?.extract()?; - let mut yaml_files: Vec = - discovered_files(&native_field(&discovered, "source_files")?)?; - yaml_files.extend(discovered_files(&native_field( - &discovered, - "schema_files", - )?)?); - Ok(RefactorFacts { - project_dir: native_field(project, "project_dir")?.str()?.extract()?, - dialect: dialect - .filter(|dialect| !dialect.is_empty()) - .unwrap_or_else(|| GENERIC_DIALECT.to_owned()), - models: model_facts(project)?.unwrap_or_default(), - sources: resource_columns(&native_field(&compiled, "sources")?, "source_entry")?, - seeds: resource_columns(&native_field(&compiled, "seeds")?, "schema_entry")?, - model_files: discovered_files(&native_field(&discovered, "model_files")?)?, - authored_files: authored_files(&discovered)?, - yaml_files, - unit_tests: unit_tests(&compiled)?, - python_locations: locations, - }) -} - /// Every compiled SQL unit test with its file, header name and target models. -fn unit_tests(compiled: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn unit_tests(compiled: &Bound<'_, PyAny>) -> PyResult> { let mut tests: Vec = Vec::new(); for test in native_field(compiled, "sql_tests")?.try_iter()? { let test: Bound<'_, PyAny> = test?; @@ -376,20 +310,6 @@ fn identity_key(identity: &Bound<'_, PyAny>) -> PyResult { identity.repr()?.extract() } -/// Scoped declarations the test file at `path` uses that resolve differently from `destination`. -pub(crate) fn unit_test_rebindings( - project: &Bound<'_, PyAny>, - paths: (&str, &str), -) -> PyResult>> { - let py: Python<'_> = project.py(); - let index: Bound<'_, PyAny> = native_field( - &native_field(&native_field(project, "graph")?, "project")?, - "scope_index", - )?; - let lookup: Bound<'_, PyAny> = build_scope_lookup(py, &index)?.into_bound(py); - scope_rebound_usages(py, &lookup, paths) -} - /// Where scope placement puts declarations once every `(model, destination)` moves together. pub(crate) fn declaration_placement( project: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python-front/src/refactor/main/compare_contracts.rs b/crates/sqlbuild-python-front/src/refactor/main/compare_contracts.rs index ef10768f9f..7a56513c46 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/compare_contracts.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/compare_contracts.rs @@ -1,10 +1,8 @@ //! Selected declarations compared with one bulk read of physical columns. use crate::refactor::_helpers::contract_compare; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use pyo3::prelude::{Bound, PyResult}; +use pyo3::types::{PyDict, PyTuple}; /// Selected declarations compared with one bulk read of physical columns. #[inline] diff --git a/crates/sqlbuild-python-front/src/refactor/main/declaration_placement.rs b/crates/sqlbuild-python-front/src/refactor/main/declaration_placement.rs index 14ff083598..806dd00a21 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/declaration_placement.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/declaration_placement.rs @@ -1,9 +1,7 @@ //! Where scope placement puts declarations once every `(model, destination)` moves together. use crate::refactor::_helpers::refactor_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_refactor::refactoring::models::DeclarationPlacement; use std::collections::BTreeSet; diff --git a/crates/sqlbuild-python-front/src/refactor/main/edit_contract_model.rs b/crates/sqlbuild-python-front/src/refactor/main/edit_contract_model.rs index c924eea433..faeec2ffd9 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/edit_contract_model.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/edit_contract_model.rs @@ -1,18 +1,58 @@ //! The model's MODEL header with physical columns adopted, or the ownership conflict. -use crate::refactor::_helpers::contract_edits; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::refactor::_helpers::contract_edits::EditRow; +use crate::refactor::_helpers::contract_edits::edit_outcome; +use crate::refactor::_helpers::contract_edits::edit_row; +use crate::refactor::_helpers::contract_edits::physical_rows; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_refactor::refactoring::main::edit_model_contract::edit_model_contract; +use sqlbuild_refactor::refactoring::models::{ContractModelColumn, ContractModelRequest}; /// The model's MODEL header with physical columns adopted, or the ownership conflict. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, model, physical_columns, overwrite))] pub fn edit_contract_model( py: Python<'_>, model: &Bound<'_, PyAny>, physical_columns: &Bound<'_, PyAny>, overwrite: bool, ) -> PyResult<(Option, Option)> { - contract_edits::edit_contract_model(py, model, physical_columns, overwrite) + let schema: Bound<'_, PyAny> = native_field(model, "schema_entry")?; + let mut columns: Vec = Vec::new(); + let mut dynamic_patterns: Vec> = Vec::new(); + let shared_schema: bool = + !schema.is_none() && !native_field(&schema, "model_schema")?.is_none(); + if !schema.is_none() { + for column in native_field(&schema, "columns")?.try_iter()? { + let column: Bound<'_, PyAny> = column?; + let location: Bound<'_, PyAny> = native_field(&column, "location")?; + let location_path: Option = if location.is_none() { + None + } else { + Some(native_field(&location, "path")?.str()?.extract()?) + }; + columns.push(ContractModelColumn { + name: native_field(&column, "name")?.extract()?, + declared_type: native_field(&column, "type")?.extract()?, + location_path, + }); + } + for family in native_field(&schema, "dynamic_columns")?.try_iter()? { + dynamic_patterns.push(native_field(&family?, "name_pattern")?.extract()?); + } + } + let request = ContractModelRequest { + contents: native_field(model, "authored_sql")?.extract()?, + relative_path: native_field(model, "relative_path")?.str()?.extract()?, + shared_schema, + columns, + dynamic_patterns, + physical_columns: physical_rows(physical_columns)?, + overwrite, + }; + let row: EditRow = compiler_guard(|| edit_row(edit_model_contract(&request)))?; + edit_outcome(py, row) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/edit_contract_source.rs b/crates/sqlbuild-python-front/src/refactor/main/edit_contract_source.rs index ca1b85b4d6..13755fd84e 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/edit_contract_source.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/edit_contract_source.rs @@ -1,17 +1,43 @@ //! The source's YAML declaration with physical columns adopted, or the ownership conflict. -use crate::refactor::_helpers::contract_edits; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::refactor::_helpers::contract_edits::EditRow; +use crate::refactor::_helpers::contract_edits::edit_outcome; +use crate::refactor::_helpers::contract_edits::edit_row; +use crate::refactor::_helpers::contract_edits::physical_rows; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_refactor::refactoring::main::edit_source_contract::edit_source_contract; +use sqlbuild_refactor::refactoring::models::ContractSourceRequest; /// The source's YAML declaration with physical columns adopted, or the ownership conflict. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, source, physical_columns, overwrite, contents = None))] pub fn edit_contract_source( source: &Bound<'_, PyAny>, physical_columns: &Bound<'_, PyAny>, overwrite: bool, contents: Option, ) -> PyResult<(Option, Option)> { - contract_edits::edit_contract_source(source, physical_columns, overwrite, contents) + let discovered_contents: String = + native_field(&native_field(source, "source_file")?, "contents")?.extract()?; + let mut columns: Vec<(String, Option)> = Vec::new(); + for column in native_field(&native_field(source, "source_entry")?, "columns")?.try_iter()? { + let column: Bound<'_, PyAny> = column?; + columns.push(( + native_field(&column, "name")?.extract()?, + native_field(&column, "type")?.extract()?, + )); + } + let request = ContractSourceRequest { + contents: contents.unwrap_or_else(|| discovered_contents.clone()), + discovered_contents, + name: native_field(source, "name")?.extract()?, + columns, + physical_columns: physical_rows(physical_columns)?, + overwrite, + }; + let row: EditRow = compiler_guard(|| edit_row(Ok(edit_source_contract(&request))))?; + edit_outcome(source.py(), row) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/model_facts.rs b/crates/sqlbuild-python-front/src/refactor/main/model_facts.rs index 315bfc6065..1e48d9f70f 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/model_facts.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/model_facts.rs @@ -1,13 +1,37 @@ //! Every compiled model's refactoring facts, or none without a project. -use crate::refactor::_helpers::refactor_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::refactor::_helpers::refactor_facts::model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_refactor::refactoring::models::ModelFacts; +use std::collections::HashMap; /// Every compiled model's refactoring facts, or none without a project. -#[inline] pub fn model_facts(refactor_project: &Bound<'_, PyAny>) -> PyResult>> { - refactor_facts::model_facts(refactor_project) + if refactor_project.is_none() { + return Ok(None); + } + let project: Bound<'_, PyAny> = + native_field(&native_field(refactor_project, "graph")?, "project")?; + let project_dir: Bound<'_, PyAny> = native_field(refactor_project, "project_dir")?; + let mut expansions: HashMap> = HashMap::new(); + let items: Vec> = native_field(&project, "sql_expansions")? + .call_method0("items")? + .try_iter()? + .collect::>()?; + for item in items.iter().rev() { + let (path, expansion): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item.extract()?; + expansions.insert(path.call_method0("resolve")?.str()?.extract()?, expansion); + } + let mut facts: Vec = Vec::new(); + for compiled in native_field(&project, "models")?.try_iter()? { + let compiled: Bound<'_, PyAny> = compiled?; + let resolved: String = project_dir + .div(native_field(&compiled, "relative_path")?)? + .call_method0("resolve")? + .str()? + .extract()?; + facts.push(model(&compiled, expansions.get(&resolved))?); + } + Ok(Some(facts)) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/refactor_facts.rs b/crates/sqlbuild-python-front/src/refactor/main/refactor_facts.rs index dc984f9324..ca930f5dfa 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/refactor_facts.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/refactor_facts.rs @@ -1,17 +1,51 @@ //! The facts one refactoring plan of `requests` reads. -use crate::refactor::_helpers::refactor_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use sqlbuild_refactor::refactoring::models::RefactorFacts; -use sqlbuild_refactor::refactoring::models::RefactorRequest; +use crate::refactor::_helpers::refactor_facts::GENERIC_DIALECT; +use crate::refactor::_helpers::refactor_facts::authored_files; +use crate::refactor::_helpers::refactor_facts::discovered_files; +use crate::refactor::_helpers::refactor_facts::move_keeps_name; +use crate::refactor::_helpers::refactor_facts::python_locations; +use crate::refactor::_helpers::refactor_facts::resource_columns; +use crate::refactor::_helpers::refactor_facts::unit_tests; +use crate::refactor::main::model_facts::model_facts; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_refactor::refactoring::models::{ + DiscoveredFile, ManualLocation, RefactorFacts, RefactorRequest, +}; /// The facts one refactoring plan of `requests` reads. -#[inline] pub fn refactor_facts( project: &Bound<'_, PyAny>, requests: &[RefactorRequest], ) -> PyResult { - refactor_facts::refactor_facts(project, requests) + let mut locations: Vec = Vec::new(); + for request in requests { + if !move_keeps_name(request) { + locations.extend(python_locations(project, request)?); + } + } + let compiled: Bound<'_, PyAny> = native_field(&native_field(project, "graph")?, "project")?; + let discovered: Bound<'_, PyAny> = native_field(project, "discovered")?; + let dialect: Option = native_field(&compiled, "sql_analysis_dialect")?.extract()?; + let mut yaml_files: Vec = + discovered_files(&native_field(&discovered, "source_files")?)?; + yaml_files.extend(discovered_files(&native_field( + &discovered, + "schema_files", + )?)?); + Ok(RefactorFacts { + project_dir: native_field(project, "project_dir")?.str()?.extract()?, + dialect: dialect + .filter(|dialect| !dialect.is_empty()) + .unwrap_or_else(|| GENERIC_DIALECT.to_owned()), + models: model_facts(project)?.unwrap_or_default(), + sources: resource_columns(&native_field(&compiled, "sources")?, "source_entry")?, + seeds: resource_columns(&native_field(&compiled, "seeds")?, "schema_entry")?, + model_files: discovered_files(&native_field(&discovered, "model_files")?)?, + authored_files: authored_files(&discovered)?, + yaml_files, + unit_tests: unit_tests(&compiled)?, + python_locations: locations, + }) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/register_contract_compare.rs b/crates/sqlbuild-python-front/src/refactor/main/register_contract_compare.rs index bdf7573d6a..1f57377114 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/register_contract_compare.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/register_contract_compare.rs @@ -1,12 +1,13 @@ //! Register the contract compare functions and classes. -use crate::refactor::_helpers::contract_compare; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::refactor::_helpers::contract_compare::compare_contract_evidence; +use crate::refactor::_helpers::contract_compare::compare_contracts; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the contract compare functions and classes. -#[inline] pub fn register_contract_compare(module: &Bound<'_, PyModule>) -> PyResult<()> { - contract_compare::register(module) + module.add_function(wrap_pyfunction!(compare_contracts, module)?)?; + module.add_function(wrap_pyfunction!(compare_contract_evidence, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs b/crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs index 0b3046740f..09c58d09da 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs @@ -1,12 +1,15 @@ //! Register the contract edits functions and classes. -use crate::refactor::_helpers::contract_edits; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::refactor::main::edit_contract_model::edit_contract_model; +use crate::refactor::main::edit_contract_source::edit_contract_source; +use crate::refactor::main::remaining_contract_findings::remaining_contract_findings; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the contract edits functions and classes. -#[inline] pub fn register_contract_edits(module: &Bound<'_, PyModule>) -> PyResult<()> { - contract_edits::register(module) + module.add_function(wrap_pyfunction!(edit_contract_model, module)?)?; + module.add_function(wrap_pyfunction!(edit_contract_source, module)?)?; + module.add_function(wrap_pyfunction!(remaining_contract_findings, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/register_python_strings.rs b/crates/sqlbuild-python-front/src/refactor/main/register_python_strings.rs index e5f93791ba..c06586d870 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/register_python_strings.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/register_python_strings.rs @@ -1,12 +1,11 @@ //! Register the python strings functions and classes. -use crate::refactor::_helpers::python_strings; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::refactor::_helpers::python_strings::find_python_string_locations; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the python strings functions and classes. -#[inline] pub fn register_python_strings(module: &Bound<'_, PyModule>) -> PyResult<()> { - python_strings::register(module) + module.add_function(wrap_pyfunction!(find_python_string_locations, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/remaining_contract_findings.rs b/crates/sqlbuild-python-front/src/refactor/main/remaining_contract_findings.rs index f8ef7ff7d7..40482ea74f 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/remaining_contract_findings.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/remaining_contract_findings.rs @@ -1,19 +1,54 @@ //! The findings an edit leaves, plus an ownership-conflict finding when the edit was refused. -use crate::refactor::_helpers::contract_edits; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; -use pyo3::types::PyTuple; +use crate::refactor::_helpers::contract_edits::RESOLVED_KINDS; +use crate::refactor::_helpers::contract_edits::adoption_error; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// The findings an edit leaves, plus an ownership-conflict finding when the edit was refused. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, findings, overwrite, conflict))] pub fn remaining_contract_findings<'py>( py: Python<'py>, findings: &Bound<'py, PyAny>, overwrite: bool, conflict: Option, ) -> PyResult> { - contract_edits::remaining_contract_findings(py, findings, overwrite, conflict) + let findings: Vec> = findings.try_iter()?.collect::>()?; + if let Some(conflict) = conflict { + let Some(first) = findings.first() else { + return Err(adoption_error( + py, + "contract ownership conflict requires resource evidence".to_owned(), + )); + }; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("resource_type", native_field(first, "resource_type")?)?; + keywords.set_item("resource_name", native_field(first, "resource_name")?)?; + keywords.set_item( + "kind", + enum_member(py, RecordId::ContractFindingKind, "OWNERSHIP_CONFLICT")?, + )?; + keywords.set_item("message", conflict)?; + let ownership: Bound<'py, PyAny> = + new_record(py, RecordId::ContractFinding, (), Some(&keywords))?; + let mut remaining: Vec> = findings.clone(); + remaining.push(ownership); + return PyTuple::new(py, remaining); + } + let mut remaining: Vec> = Vec::new(); + for finding in findings { + let kind: String = native_field(&finding, "kind")?.str()?.extract()?; + let resolved: bool = RESOLVED_KINDS.0.contains(&kind.as_str()) + || (overwrite && RESOLVED_KINDS.1.contains(&kind.as_str())); + if !resolved { + remaining.push(finding); + } + } + PyTuple::new(py, remaining) } diff --git a/crates/sqlbuild-python-front/src/refactor/main/unit_test_rebindings.rs b/crates/sqlbuild-python-front/src/refactor/main/unit_test_rebindings.rs index 9b31ee349d..72624eeb1a 100644 --- a/crates/sqlbuild-python-front/src/refactor/main/unit_test_rebindings.rs +++ b/crates/sqlbuild-python-front/src/refactor/main/unit_test_rebindings.rs @@ -1,15 +1,20 @@ //! Scoped declarations the test file at `path` uses that resolve differently from `destination`. -use crate::refactor::_helpers::refactor_facts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::scopes::main::index::build_scope_lookup::build_scope_lookup; +use crate::scopes::main::reports::scope_rebound_usages::scope_rebound_usages; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Scoped declarations the test file at `path` uses that resolve differently from `destination`. -#[inline] pub fn unit_test_rebindings( project: &Bound<'_, PyAny>, paths: (&str, &str), ) -> PyResult>> { - refactor_facts::unit_test_rebindings(project, paths) + let py: Python<'_> = project.py(); + let index: Bound<'_, PyAny> = native_field( + &native_field(&native_field(project, "graph")?, "project")?, + "scope_index", + )?; + let lookup: Bound<'_, PyAny> = build_scope_lookup(py, &index)?.into_bound(py); + scope_rebound_usages(py, &lookup, paths) } diff --git a/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_arguments.rs b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_arguments.rs index b416e2e2aa..e28096d38e 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_arguments.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_arguments.rs @@ -2,11 +2,9 @@ use pyo3::PyErr; use pyo3::exceptions::PyKeyError; -use pyo3::prelude::{ - Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyModule, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyList, PyListMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_render::macro_arguments::errors::ArgumentError; use sqlbuild_render::macro_arguments::main::parse_macro_arguments::parse_macro_arguments as parse; use sqlbuild_render::macro_arguments::models::{ArgumentValue, MacroArguments}; @@ -62,7 +60,7 @@ impl ArgumentHost for UnicodeHost<'_> { /// `("ok", positional, keywords, typed references)` or `("error", detail, help, line, column)`. #[pyfunction] -fn parse_macro_arguments( +pub(crate) fn parse_macro_arguments( py: Python<'_>, text: &str, nested: Vec<(usize, usize)>, @@ -146,8 +144,3 @@ fn value_row<'py>(py: Python<'py>, value: &ArgumentValue) -> PyResult) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_macro_arguments, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_rows.rs b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_rows.rs index 6d5b316bd9..ce05b3c248 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_rows.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_rows.rs @@ -1,10 +1,7 @@ //! Python rows of macro call sites and splice spans. -use crate::render::models::SpanTuple; -use crate::render::types::SpanRow; -use sqlbuild_analysis::semantic_validation::models::Expansion; use sqlbuild_python_base::positional_record; -use sqlbuild_render::macro_calls::models::{MacroCallSite, MacroSpliceSpan}; +use sqlbuild_render::macro_calls::models::MacroCallSite; positional_record! { into; @@ -19,26 +16,6 @@ positional_record! { } } -/// A span as the tuple Python reads. -pub(crate) fn span_tuple(span: &SpanRow) -> SpanTuple { - SpanTuple { - source_start: span.source_start, - source_end: span.source_end, - output_start: span.output_start, - output_end: span.output_end, - } -} - -/// A span as the expansion semantic validation maps offsets through. -pub(crate) fn span_expansion(span: SpanRow) -> Expansion { - Expansion { - source_start: span.source_start, - source_end: span.source_end, - output_start: span.output_start, - output_end: span.output_end, - } -} - pub(crate) fn site_row(site: MacroCallSite) -> SiteRow { SiteRow { start: site.start, @@ -49,12 +26,3 @@ pub(crate) fn site_row(site: MacroCallSite) -> SiteRow { typed_reference_text: site.typed_reference_text, } } - -pub(crate) fn span_row(span: MacroSpliceSpan) -> SpanRow { - SpanRow { - source_start: span.source_start, - source_end: span.source_end, - output_start: span.output_start, - output_end: span.output_end, - } -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_calls.rs b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_calls.rs index 3a574a0d63..584ff42afd 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_calls.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/calls/macro_calls.rs @@ -1,28 +1,22 @@ //! Macro call-site scanning and splicing for the Python macro bridge, with the GIL released. +use crate::render::main::calls::span_row::span_row; +use crate::render::main::calls::span_tuple::span_tuple; +use crate::render::main::calls::top_level_macro_call_names::top_level_macro_call_names; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyTuple; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_render::macro_calls::main::macro_call_names::macro_call_names as scan_names; use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites as scan_sites; use sqlbuild_render::macro_calls::main::splice_macro_calls::splice_macro_calls as splice_calls; use sqlbuild_render::macro_calls::models::{MacroCallScan, MacroScanFailure}; -use crate::render::_helpers::calls::macro_call_memo::MacroCallMemo; -use crate::render::_helpers::calls::macro_call_rows::SiteRow; -use crate::render::_helpers::calls::macro_call_rows::site_row; -use crate::render::_helpers::calls::macro_call_rows::span_row; -use crate::render::_helpers::calls::macro_call_rows::span_tuple; -use crate::render::_helpers::expansion::macro_expansion::raised; +use crate::render::_helpers::calls::macro_call_rows::{SiteRow, site_row}; use crate::render::models::SpanTuple; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::types::CompilerDetach; -use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -const DECLARATION_CALLS: [&str; 2] = ["enum", "const"]; +pub(crate) const DECLARATION_CALLS: [&str; 2] = ["enum", "const"]; /// `(call start or None, call name or None, message)` of the error Python's scan raises. type FailureRow = (Option, Option, &'static str); @@ -48,33 +42,12 @@ pub(crate) fn scan_macro_call_sites( )) } -/// Python's `find_macro_call_names`: top-level call names in encounter order, declarations excluded. -pub(crate) fn top_level_macro_call_names<'py>( +/// Unique authored macro call names in encounter order, declaration calls excluded. +#[pyfunction] +pub(crate) fn find_macro_call_names<'py>( py: Python<'py>, sql: &str, - syntax: Option<&LexicalSyntax>, ) -> PyResult> { - let names = py - .compiler_detach(|| Ok(scan_names(sql, false, syntax))) - .map_err(compiler_error)?; - match names { - Ok(names) => PyTuple::new( - py, - names - .into_iter() - .filter(|name| !DECLARATION_CALLS.contains(&name.as_str())) - .collect::>(), - ), - Err(error) => Err(raised( - &error_class(py, ErrorId::CompileInput)?, - error.message(), - )), - } -} - -/// Unique authored macro call names in encounter order, declaration calls excluded. -#[pyfunction] -fn find_macro_call_names<'py>(py: Python<'py>, sql: &str) -> PyResult> { top_level_macro_call_names(py, sql, None) } @@ -99,11 +72,3 @@ pub(crate) fn splice_macro_calls( .collect(), )) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scan_macro_call_sites, module)?)?; - module.add_function(wrap_pyfunction!(find_macro_call_names, module)?)?; - module.add_function(wrap_pyfunction!(splice_macro_calls, module)?)?; - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/declarations/authored_sql.rs b/crates/sqlbuild-python-front/src/render/_helpers/declarations/authored_sql.rs index 386a946494..46cb26fb9f 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/declarations/authored_sql.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/declarations/authored_sql.rs @@ -1,35 +1,31 @@ //! Authored SQL expanded through project vars, declarations and macros, with each pass's spans. +use crate::render::main::calls::span_row::span_row; +use crate::render::main::calls::span_tuple::span_tuple; +use crate::render::main::declarations::expand_with_inputs::expand_with_inputs; +use crate::render::main::declarations::span_rows::span_rows; use crate::render::models::AuthoredExpansion; use crate::render::models::SqlExpansionInputs; -use pyo3::prelude::{ - Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::model_config::main::config::record_reads::record_reads; use crate::model_config::main::model_nodes::unicode_environment::unicode_environment; -use crate::render::_helpers::calls::macro_call_rows::span_row; -use crate::render::_helpers::calls::macro_call_rows::span_tuple; use crate::render::_helpers::declarations::declaration_expansion::expand_declarations; -use crate::render::_helpers::declarations::declaration_expansion::span_rows; use crate::render::_helpers::declarations::declaration_values::resolved_declaration_context; use crate::render::_helpers::expansion::macro_expansion_result::macro_output; use crate::render::models::ConstantRendering; -use crate::render::models::DeclarationExpansion; use crate::render::models::DeclarationExpansionClasses; -use crate::render::models::ExpandedDeclarations; -use crate::render::models::ExpansionConsumer; use crate::render::models::ExpansionRequest; use crate::render::models::SqlMacroSource; +use crate::render::models::{DeclarationExpansion, ExpandedDeclarations, ExpansionConsumer}; use crate::render::types::SpanRow; use crate::sqltext::main::declaration_scan::declaration_scan; use crate::sqltext::main::interpolation_batch::interpolation_batch; use crate::sqltext::main::optional_record_syntax::optional_record_syntax; -use crate::sqltext::models::InterpolationRow; -use crate::sqltext::models::InterpolationSources; +use crate::sqltext::models::{InterpolationRow, InterpolationSources}; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -185,20 +181,20 @@ fn declarations_expanded<'py>( } /// What one authored SQL expansion reads. -struct AuthoredSources<'a, 'py> { - sql: &'a Bound<'py, PyAny>, - file_path: &'a Bound<'py, PyAny>, - effective_vars: &'a Bound<'py, PyAny>, - context_values: &'a Bound<'py, PyAny>, - declarations: &'a Bound<'py, PyAny>, - value_renderer: &'a Bound<'py, PyAny>, - collection_rendering: &'a Bound<'py, PyAny>, - loaded_macros: &'a Bound<'py, PyAny>, - macro_context: &'a Bound<'py, PyAny>, - declaration_resolver: Option>, +pub(crate) struct AuthoredSources<'a, 'py> { + pub(crate) sql: &'a Bound<'py, PyAny>, + pub(crate) file_path: &'a Bound<'py, PyAny>, + pub(crate) effective_vars: &'a Bound<'py, PyAny>, + pub(crate) context_values: &'a Bound<'py, PyAny>, + pub(crate) declarations: &'a Bound<'py, PyAny>, + pub(crate) value_renderer: &'a Bound<'py, PyAny>, + pub(crate) collection_rendering: &'a Bound<'py, PyAny>, + pub(crate) loaded_macros: &'a Bound<'py, PyAny>, + pub(crate) macro_context: &'a Bound<'py, PyAny>, + pub(crate) declaration_resolver: Option>, } -fn authored_expansion(sources: AuthoredSources<'_, '_>) -> PyResult { +pub(crate) fn authored_expansion(sources: AuthoredSources<'_, '_>) -> PyResult { let py: Python<'_> = sources.sql.py(); let syntax: Option = context_syntax(sources.macro_context)?; let (interpolated, interpolation_spans) = substituted( @@ -283,39 +279,17 @@ fn authored_sql_with_spans<'py>(request: &Request<'_, 'py>) -> PyResult, -) -> PyResult { - let py: Python<'_> = file_path.py(); - let resolver: Bound<'_, PyAny> = inputs.declaration_resolver.bind(py).clone(); - authored_expansion(AuthoredSources { - sql: &PyString::new(py, sql).into_any(), - file_path, - effective_vars: inputs.effective_vars.bind(py), - context_values: &py.None().into_bound(py), - declarations: &input_declarations(inputs, file_path)?, - value_renderer: inputs.value_renderer.bind(py), - collection_rendering: inputs.collection_rendering.bind(py), - loaded_macros: inputs.loaded_macros.bind(py), - macro_context: inputs.macro_context.bind(py), - declaration_resolver: (!resolver.is_none()).then_some(resolver), - }) -} - /// Expand authored SQL, returning each pass's substitution spans in order. #[pyfunction] #[pyo3(signature = (**request))] -fn expand_authored_sql_with_spans<'py>( +pub(crate) fn expand_authored_sql_with_spans<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { authored_sql_with_spans(&Request::new("expand_authored_sql_with_spans", request)?) } /// The declarations visible to `file_path`: the resolver's, else the inputs', plus model-local. -fn input_declarations<'py>( +pub(crate) fn input_declarations<'py>( inputs: &SqlExpansionInputs, file_path: &Bound<'py, PyAny>, ) -> PyResult> { @@ -363,7 +337,7 @@ fn input_declarations<'py>( /// Expand authored SQL with a project's expansion context and report each pass's spans. #[pyfunction] #[pyo3(signature = (*, sql, file_path, context))] -fn expand_sql_with_spans<'py>( +pub(crate) fn expand_sql_with_spans<'py>( sql: &str, file_path: &Bound<'py, PyAny>, context: &Bound<'py, PyAny>, @@ -378,7 +352,7 @@ fn expand_sql_with_spans<'py>( /// Replace SQL interpolation tokens, returning the span of every substitution. #[pyfunction] #[pyo3(signature = (*, sql, file_path, effective_vars, context_values=None))] -fn substitute_sql_vars_with_spans<'py>( +pub(crate) fn substitute_sql_vars_with_spans<'py>( sql: &Bound<'py, PyAny>, file_path: &Bound<'py, PyAny>, effective_vars: &Bound<'py, PyAny>, @@ -391,10 +365,3 @@ fn substitute_sql_vars_with_spans<'py>( substituted(sql, file_path, (effective_vars, &context_values), None)?; Ok((substituted_sql, span_objects(py, &spans)?)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(expand_authored_sql_with_spans, module)?)?; - module.add_function(wrap_pyfunction!(expand_sql_with_spans, module)?)?; - module.add_function(wrap_pyfunction!(substitute_sql_vars_with_spans, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_expansion.rs b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_expansion.rs index a641972038..aab5b52137 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_expansion.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_expansion.rs @@ -3,7 +3,6 @@ use crate::render::models::DeclarationExpansion; use crate::render::models::DeclarationExpansionClasses; use crate::render::models::ExpandedDeclarations; -use crate::render::types::SpanRow; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyInt, PyTuple}; use sqlbuild_analysis::compiled_project::errors::DeclarationError; @@ -15,9 +14,9 @@ use sqlbuild_analysis::compiled_project::models::{ }; use sqlbuild_analysis::sql_macros::models::{ExpandedText, TypedCall}; -use crate::render::_helpers::sql_macros::sql_macro_host::expanded_sql_macro_calls; -use crate::render::_helpers::sql_macros::sql_macro_host::is_sql_macro_reference; -use crate::render::_helpers::sql_macros::sql_macro_host::reject_bare_macro_calls; +use crate::render::_helpers::sql_macros::sql_macro_host::{ + expanded_sql_macro_calls, is_sql_macro_reference, reject_bare_macro_calls, +}; use crate::sqltext::models::ScannedReference; const ENUM_REFERENCE_KIND: u8 = 0; @@ -329,16 +328,3 @@ pub(crate) fn compile_error( Err(failure) => failure, } } - -pub(crate) fn span_rows(spliced: &SplicedSql) -> Vec { - spliced - .spans - .iter() - .map(|span| SpanRow { - source_start: span.source_start, - source_end: span.source_end, - output_start: span.output_start, - output_end: span.output_end, - }) - .collect() -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_indexes.rs b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_indexes.rs index 07a9f4a102..5c186d30cb 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_indexes.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_indexes.rs @@ -3,26 +3,23 @@ use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::collections::HashMap; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::ErrorId; /// `CompileInputError(message)`. -fn compile_input_error(py: Python<'_>, message: String) -> PyErr { +pub(crate) fn compile_input_error(py: Python<'_>, message: String) -> PyErr { new_plain_error(py, ErrorId::CompileInput, &message) } -fn text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { +pub(crate) fn text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { Ok(native_field(owner, name)?.str()?.to_string()) } /// Add each declaration of `files` to `index`, rejecting a name declared twice. -fn index_declarations<'py>( +pub(crate) fn index_declarations<'py>( files: &Bound<'py, PyAny>, kind: &str, index: &Bound<'py, PyDict>, @@ -49,36 +46,23 @@ fn index_declarations<'py>( Ok(()) } -/// Every public enum and constant by name, collision-checked. -#[pyfunction] -pub(crate) fn public_declaration_indexes<'py>( - inputs: Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { - let py: Python<'py> = inputs.py(); - let enums: Bound<'py, PyDict> = PyDict::new(py); - let constants: Bound<'py, PyDict> = PyDict::new(py); - index_declarations(&native_field(&inputs, "enum_files")?, "enum", &enums)?; - index_declarations( - &native_field(&inputs, "constant_files")?, - "constant", - &constants, - )?; - Ok((enums, constants)) -} - /// Resolving `extends` chains across the authored public model schemas. -struct SchemaResolution<'py> { - py: Python<'py>, - authored: Vec<(String, Bound<'py, PyAny>)>, - by_name: HashMap, - resolved: HashMap>, +pub(crate) struct SchemaResolution<'py> { + pub(crate) py: Python<'py>, + pub(crate) authored: Vec<(String, Bound<'py, PyAny>)>, + pub(crate) by_name: HashMap, + pub(crate) resolved: HashMap>, /// Names in the order Python's `resolved` dict gains them. - order: Vec, + pub(crate) order: Vec, } impl<'py> SchemaResolution<'py> { /// Python's `_resolve_model_schema_declaration`. - fn resolve(&mut self, name: &str, resolving: &[String]) -> PyResult> { + pub(crate) fn resolve( + &mut self, + name: &str, + resolving: &[String], + ) -> PyResult> { if let Some(existing) = self.resolved.get(name) { return Ok(existing.clone()); } @@ -165,52 +149,3 @@ fn lowered_name(column: &Bound<'_, PyAny>) -> PyResult { .call_method0("lower")? .extract() } - -/// Every public model schema by name, with its inherited columns first, collision-checked. -#[pyfunction] -pub(crate) fn public_model_schema_index<'py>( - inputs: Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = inputs.py(); - let mut authored: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); - let mut by_name: HashMap = HashMap::new(); - for file in native_field(&inputs, "model_schema_files")?.try_iter()? { - for declaration in native_field(&file?, "declarations")?.try_iter()? { - let declaration: Bound<'py, PyAny> = declaration?; - let name: String = native_field(&declaration, "name")?.extract()?; - if let Some(index) = by_name.get(&name) { - return Err(compile_input_error( - py, - format!( - "Duplicate public schema '{name}' in {} and {}", - text(&authored[*index].1, "relative_path")?, - text(&declaration, "relative_path")? - ), - )); - } - by_name.insert(name.clone(), authored.len()); - authored.push((name, declaration)); - } - } - let names: Vec = authored.iter().map(|(name, _)| name.clone()).collect(); - let mut resolution: SchemaResolution<'py> = SchemaResolution { - py, - authored, - by_name, - resolved: HashMap::new(), - order: Vec::new(), - }; - let index: Bound<'py, PyDict> = PyDict::new(py); - for name in &names { - resolution.resolve(name, &[])?; - } - for name in &resolution.order { - index.set_item(name, &resolution.resolved[name])?; - } - Ok(index) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(public_declaration_indexes, module)?)?; - module.add_function(wrap_pyfunction!(public_model_schema_index, module)?) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs index 73f68d7474..c66351e5bc 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs @@ -1,21 +1,14 @@ //! Declaration scope resolvers, per-path declaration contexts and adapter-rendered constants. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyInt, PyList, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; -use sqlbuild_host::entries::models::ExecutorHost; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction}; -use crate::scopes::main::index::build_scope_lookup::build_scope_lookup; use crate::scopes::models::NativeDeclarationContexts; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; @@ -29,7 +22,7 @@ use sqlbuild_python_base::records::types::RecordId; const NO_DECLARATION_KIND: u8 = 255; /// Declaration kinds by their native code: enum 0, constant 1, macro 2. const DECLARATION_KINDS: [&str; 3] = ["ENUM", "CONSTANT", "MACRO"]; -const TYPES_MODULE: &str = "types"; +pub(crate) const TYPES_MODULE: &str = "types"; const SCALAR_KINDS: [&str; 6] = ["string", "integer", "boolean", "float", "decimal", "null"]; const COLLECTION_KINDS: [&str; 2] = ["list", "set"]; const ARRAY_RENDERING: &str = "array"; @@ -51,7 +44,7 @@ fn declaration_identity<'py>( } /// Each project declaration's identity mapped to its process-local value. -fn runtime_declarations<'py>( +pub(crate) fn runtime_declarations<'py>( discovered: &Bound<'py, PyAny>, loaded_macros: &Bound<'py, PyAny>, ) -> PyResult> { @@ -119,7 +112,9 @@ fn runtime_declarations<'py>( } /// Resources whose declarations differ from their directory's: private owners and grantees. -fn resource_specific<'py>(scope_index: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn resource_specific<'py>( + scope_index: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = scope_index.py(); let private: Bound<'py, PyAny> = enum_member(py, RecordId::ScopeKind, "PRIVATE")?; let mut resources: Vec> = Vec::new(); @@ -137,45 +132,6 @@ fn resource_specific<'py>(scope_index: &Bound<'py, PyAny>) -> PyResult( - discovered: &Bound<'py, PyAny>, - scope_index: &Bound<'py, PyAny>, - (loaded_macros, lookup): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - let py: Python<'py> = discovered.py(); - let declarations: Bound<'py, PyDict> = runtime_declarations(discovered, loaded_macros)?; - let lookup: Bound<'py, PyAny> = if lookup.is_none() { - build_scope_lookup(py, scope_index)?.into_bound(py) - } else { - lookup.clone() - }; - let projection: Bound<'py, PyAny> = new_record_with( - py, - RecordId::DeclarationRuntimeProjection, - &[( - "declarations", - imported(py, TYPES_MODULE, "MappingProxyType")?.call1((&declarations,))?, - )], - )?; - let native_contexts: Bound<'py, PyAny> = - native_declaration_contexts(&lookup, declarations.as_any())?; - new_record_with( - py, - RecordId::DeclarationScopeResolver, - &[ - ("project_dir", native_field(discovered, "project_dir")?), - ("lookup", lookup), - ("projection", projection), - ( - "resource_specific", - resource_specific(scope_index)?.into_any(), - ), - ("native_contexts", native_contexts), - ], - ) -} - /// The native code of declaration kind `kind`, or the no-kind code. fn kind_code(kinds: &[Bound<'_, PyAny>], kind: &Bound<'_, PyAny>) -> u8 { for (code, known) in (0_u8..).zip(kinds.iter()) { @@ -331,7 +287,7 @@ pub(crate) fn native_declaration_contexts<'py>( /// Hand one scope lookup's positions and runtime values to the native classifier. #[pyfunction] #[pyo3(signature = (*, lookup, declarations))] -fn build_native_declaration_contexts<'py>( +pub(crate) fn build_native_declaration_contexts<'py>( lookup: &Bound<'py, PyAny>, declarations: &Bound<'py, PyAny>, ) -> PyResult> { @@ -353,7 +309,7 @@ pub(crate) fn resolved_declaration_context<'py>( } /// The adapter's SQL for one constant under `selected` collection rendering, or its error. -fn rendered_constant<'py>( +pub(crate) fn rendered_constant<'py>( declaration: &Bound<'py, PyAny>, value_renderer: &Bound<'py, PyAny>, selected: &Bound<'py, PyAny>, @@ -390,73 +346,13 @@ fn rendered_constant<'py>( Ok(rendered) } -/// Render one validated constant with the active adapter's typed-value contract. -pub(crate) fn constant_declaration_sql<'py>( - declaration: &Bound<'py, PyAny>, - value_renderer: &Bound<'py, PyAny>, - (collection_rendering, file_path): (&Bound<'py, PyAny>, Option<&str>), -) -> PyResult { - let py: Python<'py> = declaration.py(); - let render_as: Bound<'py, PyAny> = native_field(declaration, "render_as")?; - let selected: Bound<'py, PyAny> = if render_as.is_truthy()? { - render_as - } else { - collection_rendering.clone() - }; - let error: PyErr = match rendered_constant(declaration, value_renderer, &selected) { - Ok(rendered) => return Ok(rendered), - Err(error) => error, - }; - let rendering_error: Bound<'py, PyAny> = host_entry(py, ExecutorHost::SqlValueRenderingError)?; - let validation_error: Bound<'py, PyAny> = native_attr(py, NativeAttr::SqlValueValidationError)?; - let raised: Bound<'py, PyAny> = error.value(py).clone().into_any(); - if !(raised.is_instance(&rendering_error)? || raised.is_instance(&validation_error)?) { - return Err(error); - } - let relative_path: Bound<'py, PyAny> = native_field(declaration, "relative_path")?; - let location: String = match file_path { - Some(file_path) => file_path.to_owned(), - None => relative_path.str()?.to_string(), - }; - let compile_error: Bound<'py, PyAny> = - error_class(py, ErrorId::CompileInput)?.call1((format!( - "{} constant '{}' could not be rendered in '{}' by adapter '{}' as {}: {}", - relative_path.str()?, - native_field(declaration, "name")?.str()?, - location, - native_field(value_renderer, "adapter_name")?.str()?, - native_field(&selected, "value")?.str()?, - raised.str()?, - ),))?; - compile_error.setattr("__cause__", &raised)?; - Err(PyErr::from_value(compile_error)) -} - -/// One enum member value as a SQL scalar literal. -#[pyfunction] -#[pyo3(signature = (*, value))] -pub(crate) fn render_enum_member_value(value: &Bound<'_, PyAny>) -> PyResult { - if value.is_instance_of::() { - return value.str()?.extract(); - } - let text: String = value.extract()?; - Ok(format!("'{}'", text.replace('\'', "''"))) -} - /// Canonical visibility for one authored path, projected onto runtime declaration values. #[pyfunction] #[pyo3(signature = (*, resolver, file_path, resource=None))] -fn resolve_declaration_context<'py>( +pub(crate) fn resolve_declaration_context<'py>( resolver: &Bound<'py, PyAny>, file_path: &Bound<'py, PyAny>, resource: Option>, ) -> PyResult> { resolved_declaration_context(resolver, file_path, resource.as_ref()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_native_declaration_contexts, module)?)?; - module.add_function(wrap_pyfunction!(render_enum_member_value, module)?)?; - module.add_function(wrap_pyfunction!(resolve_declaration_context, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs index ce9fa04516..5c73a741c1 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs @@ -15,8 +15,7 @@ use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_s use sqlbuild_render::macro_calls::models::{MacroCallScan, MacroCallSite}; use crate::render::_helpers::calls::macro_arguments::parsed_arguments; -use crate::render::_helpers::expansion::macro_expansion::ExpansionClasses; -use crate::render::_helpers::expansion::macro_expansion::raised; +use crate::render::_helpers::expansion::macro_expansion::{ExpansionClasses, raised}; use crate::render::_helpers::expansion::macro_expansion_facts::ResourceRef; use crate::render::models::NativeMacroExpansion; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs index 9c23011592..69cd824d73 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs @@ -6,9 +6,7 @@ use sqlbuild_python_base::positional_record; use pyo3::FromPyObject; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyRef, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList}; use sqlbuild_render::macro_calls::errors::ScanError; use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites as scan_sites; @@ -16,11 +14,8 @@ use sqlbuild_render::macro_calls::main::splice_macro_calls::splice_macro_calls a use sqlbuild_render::macro_calls::models::{MacroCallScan, MacroCallSite}; use crate::render::_helpers::calls::macro_call_classes::CallClass; -use crate::render::_helpers::calls::macro_call_memo::MacroCallMemo; -use crate::render::_helpers::calls::macro_call_memo::poisoned; -use crate::render::_helpers::expansion::macro_evaluation::CallText; -use crate::render::_helpers::expansion::macro_evaluation::Evaluation; -use crate::render::_helpers::expansion::macro_evaluation::evaluate_call; +use crate::render::_helpers::calls::macro_call_memo::{MacroCallMemo, poisoned}; +use crate::render::_helpers::expansion::macro_evaluation::{CallText, Evaluation, evaluate_call}; use crate::render::_helpers::expansion::macro_expansion_facts::ResourceRef; use crate::render::_helpers::expansion::macro_generated_references::GeneratedReferenceClasses; use crate::render::models::SpanTuple; @@ -254,11 +249,6 @@ pub(crate) fn raised(class: &Bound<'_, PyAny>, message: &str) -> PyErr { } } -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} - /// Whether a macro's result calls any reference function. pub(crate) fn result_has_markers(classes: &ExpansionClasses<'_>, result: &str) -> bool { classes diff --git a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_result.rs b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_result.rs index 83221cb5ba..327101a06a 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_result.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_result.rs @@ -1,20 +1,19 @@ //! Expand one authored SQL string's macros into a `MacroExpansionResult`. +use crate::render::main::calls::span_row::span_row; +use crate::render::main::expansion::macro_expansion_classes::macro_expansion_classes; use crate::render::models::ExpansionRequest; use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::UserCodeHost; use crate::discovery::main::failures::explicit_references_help::explicit_references_help; -use crate::render::_helpers::calls::macro_call_rows::span_row; use crate::render::_helpers::declarations::declaration_values::resolved_declaration_context; use crate::render::_helpers::expansion::macro_expansion::expanded; -use crate::render::models::ExpansionOutput; -use crate::render::models::NativeMacroExpansion; -use crate::render::models::StateInput; +use crate::render::models::{ExpansionOutput, NativeMacroExpansion, StateInput}; use crate::render::types::SpanRow; use crate::sqltext::main::extract_references::extract_references; use sqlbuild_python_base::boundary::main::imported::imported; @@ -48,7 +47,7 @@ const BUILTINS_MODULE: &str = "builtins"; const MACRO_TOKEN: &str = "@"; const GENERATED_ALLOWED: &str = "macro-generated references"; -static EXPANSION_CLASSES: PyOnceLock> = PyOnceLock::new(); +pub(crate) static EXPANSION_CLASSES: PyOnceLock> = PyOnceLock::new(); fn generated_classes(py: Python<'_>) -> PyResult> { let help: Bound<'_, PyAny> = @@ -82,7 +81,7 @@ fn generated_classes(py: Python<'_>) -> PyResult> { ) } -fn built_classes(py: Python<'_>) -> PyResult> { +pub(crate) fn built_classes(py: Python<'_>) -> PyResult> { let declaration_kind: Bound<'_, PyAny> = record_class(py, RecordId::DeclarationKind)?; let ref_kind: Bound<'_, PyAny> = host_entry(py, UserCodeHost::SqlResourceRefKind)?; let ref_kinds: Bound<'_, PyDict> = PyDict::new(py); @@ -143,13 +142,6 @@ fn built_classes(py: Python<'_>) -> PyResult> { ) } -/// The classes, enums and helpers the native expansion builds facts and errors from, built once. -pub(crate) fn macro_expansion_classes(py: Python<'_>) -> PyResult> { - EXPANSION_CLASSES - .get_or_try_init(py, || built_classes(py).map(Bound::unbind)) - .map(|classes| classes.bind(py).clone()) -} - /// `tuple(dict.fromkeys(items))`: the items without repeats, in first-seen order. fn unique<'py>(py: Python<'py>, items: Vec>) -> PyResult> { let seen: Bound<'py, PyDict> = PyDict::new(py); @@ -294,7 +286,3 @@ pub(crate) fn expand_sql_macros<'py>( consumer: optional(arguments.value("consumer")?), }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(expand_sql_macros, module)?) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_modules.rs b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_modules.rs index 2343604556..c1486d2617 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_modules.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_modules.rs @@ -1,7 +1,7 @@ //! Static inventory of project macro modules for the Python macro loader, with the GIL released. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_pysource::macro_modules::main::inventory_macro_modules::inventory_macro_modules as inventory; use sqlbuild_pysource::macro_modules::main::macro_load_plan::macro_load_plan; use sqlbuild_pysource::macro_modules::models::{ @@ -47,7 +47,7 @@ positional_record! { positional_record! { extract; /// `(name, path, contents, dependencies)` of one macro module to load. - struct ModuleSpecRow { + pub(crate) struct ModuleSpecRow { name: String, file_path: String, contents: String, @@ -146,7 +146,9 @@ pub(crate) fn inventory_macro_modules( /// `(synthetic root, packages shallowest first, module execution order)` of `(name, path, contents, dependencies)` rows. #[pyfunction] -fn macro_module_load_plan(modules: Vec) -> (String, Vec, Vec) { +pub(crate) fn macro_module_load_plan( + modules: Vec, +) -> (String, Vec, Vec) { let specs: Vec = modules .into_iter() .map( @@ -170,9 +172,3 @@ fn macro_module_load_plan(modules: Vec) -> (String, Vec, } = macro_load_plan(&specs); (root, packages, order) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(inventory_macro_modules, module)?)?; - module.add_function(wrap_pyfunction!(macro_module_load_plan, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs index 1d2445ca42..ef2be07911 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_host.rs @@ -28,10 +28,10 @@ use sqlbuild_analysis::sql_macros::models::{ use sqlbuild_analysis::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::render::_helpers::declarations::declaration_expansion::expand_declarations; -use crate::render::_helpers::declarations::declaration_expansion::usage_visibility; -use crate::render::models::DeclarationExpansion; -use crate::render::models::ExpandedDeclarations; +use crate::render::_helpers::declarations::declaration_expansion::{ + expand_declarations, usage_visibility, +}; +use crate::render::models::{DeclarationExpansion, ExpandedDeclarations}; use crate::sqltext::main::declaration_scan::declaration_scan; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_helped_error::new_helped_error; diff --git a/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_inventory.rs b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_inventory.rs index cd02a4a7ae..3f9f194765 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_inventory.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_inventory.rs @@ -1,7 +1,7 @@ //! SQL macro files parsed and checked when the project's macros load. -use pyo3::prelude::{Bound, PyErr, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyErr, PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_analysis::sql_macros::constants::CYCLE_CODE; use sqlbuild_analysis::sql_macros::errors::SqlMacroError; use sqlbuild_analysis::sql_macros::main::parse_sql_macro::parse_sql_macro; @@ -37,7 +37,10 @@ fn definition_row(definition: SqlMacroDefinition) -> DefinitionRow { /// Each SQL macro's parameters and calls, or the first file that is not a valid SQL macro. #[pyfunction] -fn inventory_sql_macros(py: Python<'_>, sources: Vec) -> PyResult> { +pub(crate) fn inventory_sql_macros( + py: Python<'_>, + sources: Vec, +) -> PyResult> { let mut definitions: Vec = Vec::with_capacity(sources.len()); for (name, path, contents) in &sources { definitions @@ -70,7 +73,3 @@ fn inventory_sql_macros(py: Python<'_>, sources: Vec) -> PyResult) -> PyResult<()> { - module.add_function(wrap_pyfunction!(inventory_sql_macros, module)?) -} diff --git a/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_tests.rs b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_tests.rs index 5370a6a5a7..582749d28e 100644 --- a/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_tests.rs +++ b/crates/sqlbuild-python-front/src/render/_helpers/sql_macros/sql_macro_tests.rs @@ -1,48 +1,11 @@ //! CTE-list SQL macro calls in the `WITH` list of a macro-mode SQL test. -use sqlbuild_analysis::sql_macros::main::with_list_sql_macro_calls::with_list_sql_macro_calls; -use sqlbuild_analysis::sql_macros::models::SqlMacroCall; - -/// The test's SQL with each `WITH`-list SQL macro call blanked, and the macros those calls name. -pub(crate) fn without_with_list_calls(sql: &str) -> (String, Vec) { - let calls: Vec = with_list_sql_macro_calls(sql); - if calls.is_empty() { - return (sql.to_owned(), Vec::new()); - } - let mut blanked: Vec<(usize, usize)> = Vec::new(); - for call in &calls { - blanked.push((call.start, call.end)); - let after: usize = call.end + leading_space(&sql[call.end..]); - if sql[after..].starts_with(',') { - blanked.push((after, after + 1)); - } else if ends_list(&sql[after..]) { - let before: &str = sql[..call.start].trim_end(); - if before.ends_with(',') { - blanked.push((before.len() - 1, before.len())); - } - } - } - let text: String = sql - .char_indices() - .map(|(offset, character)| { - let inside: bool = is_blanked(offset, &blanked); - if inside && !character.is_whitespace() { - ' ' - } else { - character - } - }) - .collect(); - let names: Vec = calls.into_iter().map(|call| call.name).collect(); - (text, names) -} - -fn leading_space(text: &str) -> usize { +pub(crate) fn leading_space(text: &str) -> usize { text.len() - text.trim_start().len() } /// Whether the text after a call ends the `WITH` list: nothing, a statement end or `SELECT`. -fn ends_list(text: &str) -> bool { +pub(crate) fn ends_list(text: &str) -> bool { let rest: &str = text.trim_start(); let word: String = rest .chars() @@ -52,7 +15,7 @@ fn ends_list(text: &str) -> bool { } /// Whether byte `offset` falls in one of the blanked spans. -fn is_blanked(offset: usize, blanked: &[(usize, usize)]) -> bool { +pub(crate) fn is_blanked(offset: usize, blanked: &[(usize, usize)]) -> bool { blanked .iter() .any(|(start, end)| *start <= offset && offset < *end) diff --git a/crates/sqlbuild-python-front/src/render/main/calls/register_macro_arguments.rs b/crates/sqlbuild-python-front/src/render/main/calls/register_macro_arguments.rs index 1f7be17726..4d8b07ce14 100644 --- a/crates/sqlbuild-python-front/src/render/main/calls/register_macro_arguments.rs +++ b/crates/sqlbuild-python-front/src/render/main/calls/register_macro_arguments.rs @@ -1,12 +1,11 @@ //! Register the macro arguments functions and classes. -use crate::render::_helpers::calls::macro_arguments; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::_helpers::calls::macro_arguments::parse_macro_arguments; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the macro arguments functions and classes. -#[inline] pub fn register_macro_arguments(module: &Bound<'_, PyModule>) -> PyResult<()> { - macro_arguments::register(module) + module.add_function(wrap_pyfunction!(parse_macro_arguments, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/render/main/calls/register_macro_calls.rs b/crates/sqlbuild-python-front/src/render/main/calls/register_macro_calls.rs index d44a654e62..a5133f1909 100644 --- a/crates/sqlbuild-python-front/src/render/main/calls/register_macro_calls.rs +++ b/crates/sqlbuild-python-front/src/render/main/calls/register_macro_calls.rs @@ -1,12 +1,17 @@ //! Register the macro calls functions and classes. -use crate::render::_helpers::calls::macro_calls; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::_helpers::calls::macro_call_memo::MacroCallMemo; +use crate::render::_helpers::calls::macro_calls::find_macro_call_names; +use crate::render::_helpers::calls::macro_calls::scan_macro_call_sites; +use crate::render::_helpers::calls::macro_calls::splice_macro_calls; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the macro calls functions and classes. -#[inline] pub fn register_macro_calls(module: &Bound<'_, PyModule>) -> PyResult<()> { - macro_calls::register(module) + module.add_function(wrap_pyfunction!(scan_macro_call_sites, module)?)?; + module.add_function(wrap_pyfunction!(find_macro_call_names, module)?)?; + module.add_function(wrap_pyfunction!(splice_macro_calls, module)?)?; + module.add_class::()?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/render/main/calls/span_expansion.rs b/crates/sqlbuild-python-front/src/render/main/calls/span_expansion.rs index e30ecf2a63..e95e4f952c 100644 --- a/crates/sqlbuild-python-front/src/render/main/calls/span_expansion.rs +++ b/crates/sqlbuild-python-front/src/render/main/calls/span_expansion.rs @@ -1,11 +1,14 @@ //! A span as the expansion semantic validation maps offsets through. -use crate::render::_helpers::calls::macro_call_rows; use crate::render::types::SpanRow; use sqlbuild_analysis::semantic_validation::models::Expansion; /// A span as the expansion semantic validation maps offsets through. -#[inline] pub fn span_expansion(span: SpanRow) -> Expansion { - macro_call_rows::span_expansion(span) + Expansion { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + } } diff --git a/crates/sqlbuild-python-front/src/render/main/calls/span_row.rs b/crates/sqlbuild-python-front/src/render/main/calls/span_row.rs index 5a2e19132f..9ae3ef675b 100644 --- a/crates/sqlbuild-python-front/src/render/main/calls/span_row.rs +++ b/crates/sqlbuild-python-front/src/render/main/calls/span_row.rs @@ -1,11 +1,14 @@ //! Span row. -use crate::render::_helpers::calls::macro_call_rows; use crate::render::types::SpanRow; use sqlbuild_render::macro_calls::models::MacroSpliceSpan; /// Span row. -#[inline] pub fn span_row(span: MacroSpliceSpan) -> SpanRow { - macro_call_rows::span_row(span) + SpanRow { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + } } diff --git a/crates/sqlbuild-python-front/src/render/main/calls/span_tuple.rs b/crates/sqlbuild-python-front/src/render/main/calls/span_tuple.rs index d1c2173566..c5396fb6c5 100644 --- a/crates/sqlbuild-python-front/src/render/main/calls/span_tuple.rs +++ b/crates/sqlbuild-python-front/src/render/main/calls/span_tuple.rs @@ -1,11 +1,14 @@ //! A span as the tuple Python reads. -use crate::render::_helpers::calls::macro_call_rows; use crate::render::models::SpanTuple; use crate::render::types::SpanRow; /// A span as the tuple Python reads. -#[inline] pub fn span_tuple(span: &SpanRow) -> SpanTuple { - macro_call_rows::span_tuple(span) + SpanTuple { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + } } diff --git a/crates/sqlbuild-python-front/src/render/main/calls/top_level_macro_call_names.rs b/crates/sqlbuild-python-front/src/render/main/calls/top_level_macro_call_names.rs index f740688342..302e7185de 100644 --- a/crates/sqlbuild-python-front/src/render/main/calls/top_level_macro_call_names.rs +++ b/crates/sqlbuild-python-front/src/render/main/calls/top_level_macro_call_names.rs @@ -1,18 +1,36 @@ //! Python's `find_macro_call_names`: top-level call names in encounter order, declarations excluded. -use crate::render::_helpers::calls::macro_calls; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::render::_helpers::calls::macro_calls::DECLARATION_CALLS; +use crate::render::_helpers::expansion::macro_expansion::raised; +use pyo3::prelude::{Bound, PyResult, Python}; use pyo3::types::PyTuple; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_render::macro_calls::main::macro_call_names::macro_call_names as scan_names; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// Python's `find_macro_call_names`: top-level call names in encounter order, declarations excluded. -#[inline] pub fn top_level_macro_call_names<'py>( py: Python<'py>, sql: &str, syntax: Option<&LexicalSyntax>, ) -> PyResult> { - macro_calls::top_level_macro_call_names(py, sql, syntax) + let names = py + .compiler_detach(|| Ok(scan_names(sql, false, syntax))) + .map_err(compiler_error)?; + match names { + Ok(names) => PyTuple::new( + py, + names + .into_iter() + .filter(|name| !DECLARATION_CALLS.contains(&name.as_str())) + .collect::>(), + ), + Err(error) => Err(raised( + &error_class(py, ErrorId::CompileInput)?, + error.message(), + )), + } } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs b/crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs index 98c8ffe2cd..dd4556b8eb 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs @@ -1,20 +1,53 @@ //! Render one validated constant with the active adapter's typed-value contract. -use crate::render::_helpers::declarations::declaration_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::render::_helpers::declarations::declaration_values::rendered_constant; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::registry::native_attr::native_attr; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_base::records::types::NativeAttr; /// Render one validated constant with the active adapter's typed-value contract. -#[inline] pub fn constant_declaration_sql<'py>( declaration: &Bound<'py, PyAny>, value_renderer: &Bound<'py, PyAny>, (collection_rendering, file_path): (&Bound<'py, PyAny>, Option<&str>), ) -> PyResult { - declaration_values::constant_declaration_sql( - declaration, - value_renderer, - (collection_rendering, file_path), - ) + let py: Python<'py> = declaration.py(); + let render_as: Bound<'py, PyAny> = native_field(declaration, "render_as")?; + let selected: Bound<'py, PyAny> = if render_as.is_truthy()? { + render_as + } else { + collection_rendering.clone() + }; + let error: PyErr = match rendered_constant(declaration, value_renderer, &selected) { + Ok(rendered) => return Ok(rendered), + Err(error) => error, + }; + let rendering_error: Bound<'py, PyAny> = host_entry(py, ExecutorHost::SqlValueRenderingError)?; + let validation_error: Bound<'py, PyAny> = native_attr(py, NativeAttr::SqlValueValidationError)?; + let raised: Bound<'py, PyAny> = error.value(py).clone().into_any(); + if !(raised.is_instance(&rendering_error)? || raised.is_instance(&validation_error)?) { + return Err(error); + } + let relative_path: Bound<'py, PyAny> = native_field(declaration, "relative_path")?; + let location: String = match file_path { + Some(file_path) => file_path.to_owned(), + None => relative_path.str()?.to_string(), + }; + let compile_error: Bound<'py, PyAny> = + error_class(py, ErrorId::CompileInput)?.call1((format!( + "{} constant '{}' could not be rendered in '{}' by adapter '{}' as {}: {}", + relative_path.str()?, + native_field(declaration, "name")?.str()?, + location, + native_field(value_renderer, "adapter_name")?.str()?, + native_field(&selected, "value")?.str()?, + raised.str()?, + ),))?; + compile_error.setattr("__cause__", &raised)?; + Err(PyErr::from_value(compile_error)) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/declaration_scope_resolver.rs b/crates/sqlbuild-python-front/src/render/main/declarations/declaration_scope_resolver.rs index 0f8d8e6cbd..93578455e7 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/declaration_scope_resolver.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/declaration_scope_resolver.rs @@ -1,16 +1,52 @@ //! Pair the serializable static index with original process-local declaration values. -use crate::render::_helpers::declarations::declaration_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::render::_helpers::declarations::declaration_values::TYPES_MODULE; +use crate::render::_helpers::declarations::declaration_values::native_declaration_contexts; +use crate::render::_helpers::declarations::declaration_values::resource_specific; +use crate::render::_helpers::declarations::declaration_values::runtime_declarations; +use crate::scopes::main::index::build_scope_lookup::build_scope_lookup; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// Pair the serializable static index with original process-local declaration values. -#[inline] pub fn declaration_scope_resolver<'py>( discovered: &Bound<'py, PyAny>, scope_index: &Bound<'py, PyAny>, (loaded_macros, lookup): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { - declaration_values::declaration_scope_resolver(discovered, scope_index, (loaded_macros, lookup)) + let py: Python<'py> = discovered.py(); + let declarations: Bound<'py, PyDict> = runtime_declarations(discovered, loaded_macros)?; + let lookup: Bound<'py, PyAny> = if lookup.is_none() { + build_scope_lookup(py, scope_index)?.into_bound(py) + } else { + lookup.clone() + }; + let projection: Bound<'py, PyAny> = new_record_with( + py, + RecordId::DeclarationRuntimeProjection, + &[( + "declarations", + imported(py, TYPES_MODULE, "MappingProxyType")?.call1((&declarations,))?, + )], + )?; + let native_contexts: Bound<'py, PyAny> = + native_declaration_contexts(&lookup, declarations.as_any())?; + new_record_with( + py, + RecordId::DeclarationScopeResolver, + &[ + ("project_dir", native_field(discovered, "project_dir")?), + ("lookup", lookup), + ("projection", projection), + ( + "resource_specific", + resource_specific(scope_index)?.into_any(), + ), + ("native_contexts", native_contexts), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/expand_with_inputs.rs b/crates/sqlbuild-python-front/src/render/main/declarations/expand_with_inputs.rs index 4b1bac8d3d..d2cdaade28 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/expand_with_inputs.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/expand_with_inputs.rs @@ -1,18 +1,31 @@ //! `sql` expanded with lint's expansion inputs, as `file_path`'s declarations see them. -use crate::render::_helpers::declarations::authored_sql; +use crate::render::_helpers::declarations::authored_sql::AuthoredSources; +use crate::render::_helpers::declarations::authored_sql::authored_expansion; +use crate::render::_helpers::declarations::authored_sql::input_declarations; use crate::render::models::AuthoredExpansion; use crate::render::models::SqlExpansionInputs; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; /// `sql` expanded with lint's expansion inputs, as `file_path`'s declarations see them. -#[inline] pub fn expand_with_inputs( inputs: &SqlExpansionInputs, sql: &str, file_path: &Bound<'_, PyAny>, ) -> PyResult { - authored_sql::expand_with_inputs(inputs, sql, file_path) + let py: Python<'_> = file_path.py(); + let resolver: Bound<'_, PyAny> = inputs.declaration_resolver.bind(py).clone(); + authored_expansion(AuthoredSources { + sql: &PyString::new(py, sql).into_any(), + file_path, + effective_vars: inputs.effective_vars.bind(py), + context_values: &py.None().into_bound(py), + declarations: &input_declarations(inputs, file_path)?, + value_renderer: inputs.value_renderer.bind(py), + collection_rendering: inputs.collection_rendering.bind(py), + loaded_macros: inputs.loaded_macros.bind(py), + macro_context: inputs.macro_context.bind(py), + declaration_resolver: (!resolver.is_none()).then_some(resolver), + }) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/public_declaration_indexes.rs b/crates/sqlbuild-python-front/src/render/main/declarations/public_declaration_indexes.rs index aada555fa0..77813c5753 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/public_declaration_indexes.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/public_declaration_indexes.rs @@ -1,15 +1,24 @@ //! Every public enum and constant by name, collision-checked. -use crate::render::_helpers::declarations::declaration_indexes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::render::_helpers::declarations::declaration_indexes::index_declarations; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Every public enum and constant by name, collision-checked. -#[inline] +#[pyfunction] pub fn public_declaration_indexes<'py>( inputs: Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { - declaration_indexes::public_declaration_indexes(inputs) + let py: Python<'py> = inputs.py(); + let enums: Bound<'py, PyDict> = PyDict::new(py); + let constants: Bound<'py, PyDict> = PyDict::new(py); + index_declarations(&native_field(&inputs, "enum_files")?, "enum", &enums)?; + index_declarations( + &native_field(&inputs, "constant_files")?, + "constant", + &constants, + )?; + Ok((enums, constants)) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/public_model_schema_index.rs b/crates/sqlbuild-python-front/src/render/main/declarations/public_model_schema_index.rs index 1c1d35991e..19aefe81c6 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/public_model_schema_index.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/public_model_schema_index.rs @@ -1,13 +1,52 @@ //! Every public model schema by name, with its inherited columns first, collision-checked. -use crate::render::_helpers::declarations::declaration_indexes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use crate::render::_helpers::declarations::declaration_indexes::SchemaResolution; +use crate::render::_helpers::declarations::declaration_indexes::compile_input_error; +use crate::render::_helpers::declarations::declaration_indexes::text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::HashMap; /// Every public model schema by name, with its inherited columns first, collision-checked. -#[inline] +#[pyfunction] pub fn public_model_schema_index<'py>(inputs: Bound<'py, PyAny>) -> PyResult> { - declaration_indexes::public_model_schema_index(inputs) + let py: Python<'py> = inputs.py(); + let mut authored: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); + let mut by_name: HashMap = HashMap::new(); + for file in native_field(&inputs, "model_schema_files")?.try_iter()? { + for declaration in native_field(&file?, "declarations")?.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + let name: String = native_field(&declaration, "name")?.extract()?; + if let Some(index) = by_name.get(&name) { + return Err(compile_input_error( + py, + format!( + "Duplicate public schema '{name}' in {} and {}", + text(&authored[*index].1, "relative_path")?, + text(&declaration, "relative_path")? + ), + )); + } + by_name.insert(name.clone(), authored.len()); + authored.push((name, declaration)); + } + } + let names: Vec = authored.iter().map(|(name, _)| name.clone()).collect(); + let mut resolution: SchemaResolution<'py> = SchemaResolution { + py, + authored, + by_name, + resolved: HashMap::new(), + order: Vec::new(), + }; + let index: Bound<'py, PyDict> = PyDict::new(py); + for name in &names { + resolution.resolve(name, &[])?; + } + for name in &resolution.order { + index.set_item(name, &resolution.resolved[name])?; + } + Ok(index) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs b/crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs index 1a77c6da6c..a5852cb504 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs @@ -1,12 +1,15 @@ //! Register the authored sql functions and classes. -use crate::render::_helpers::declarations::authored_sql; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::_helpers::declarations::authored_sql::expand_authored_sql_with_spans; +use crate::render::_helpers::declarations::authored_sql::expand_sql_with_spans; +use crate::render::_helpers::declarations::authored_sql::substitute_sql_vars_with_spans; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the authored sql functions and classes. -#[inline] pub fn register_authored_sql(module: &Bound<'_, PyModule>) -> PyResult<()> { - authored_sql::register(module) + module.add_function(wrap_pyfunction!(expand_authored_sql_with_spans, module)?)?; + module.add_function(wrap_pyfunction!(expand_sql_with_spans, module)?)?; + module.add_function(wrap_pyfunction!(substitute_sql_vars_with_spans, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_indexes.rs b/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_indexes.rs index dab0eed265..70395954f7 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_indexes.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_indexes.rs @@ -1,12 +1,12 @@ //! Register the declaration indexes functions and classes. -use crate::render::_helpers::declarations::declaration_indexes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::main::declarations::public_declaration_indexes::public_declaration_indexes; +use crate::render::main::declarations::public_model_schema_index::public_model_schema_index; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the declaration indexes functions and classes. -#[inline] pub fn register_declaration_indexes(module: &Bound<'_, PyModule>) -> PyResult<()> { - declaration_indexes::register(module) + module.add_function(wrap_pyfunction!(public_declaration_indexes, module)?)?; + module.add_function(wrap_pyfunction!(public_model_schema_index, module)?) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs b/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs index 92d91956ef..feb05ea17a 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs @@ -1,12 +1,15 @@ //! Register the declaration values functions and classes. -use crate::render::_helpers::declarations::declaration_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::_helpers::declarations::declaration_values::build_native_declaration_contexts; +use crate::render::_helpers::declarations::declaration_values::resolve_declaration_context; +use crate::render::main::declarations::render_enum_member_value::render_enum_member_value; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the declaration values functions and classes. -#[inline] pub fn register_declaration_values(module: &Bound<'_, PyModule>) -> PyResult<()> { - declaration_values::register(module) + module.add_function(wrap_pyfunction!(build_native_declaration_contexts, module)?)?; + module.add_function(wrap_pyfunction!(render_enum_member_value, module)?)?; + module.add_function(wrap_pyfunction!(resolve_declaration_context, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/render_enum_member_value.rs b/crates/sqlbuild-python-front/src/render/main/declarations/render_enum_member_value.rs index 18b3148a1e..fd3822fa90 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/render_enum_member_value.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/render_enum_member_value.rs @@ -1,12 +1,16 @@ //! One enum member value as a SQL scalar literal. -use crate::render::_helpers::declarations::declaration_values; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyInt; /// One enum member value as a SQL scalar literal. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, value))] pub fn render_enum_member_value(value: &Bound<'_, PyAny>) -> PyResult { - declaration_values::render_enum_member_value(value) + if value.is_instance_of::() { + return value.str()?.extract(); + } + let text: String = value.extract()?; + Ok(format!("'{}'", text.replace('\'', "''"))) } diff --git a/crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs b/crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs index 69f717b456..69bf7246ce 100644 --- a/crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs +++ b/crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs @@ -1,11 +1,18 @@ //! Span rows. -use crate::render::_helpers::declarations::declaration_expansion; use crate::render::types::SpanRow; use sqlbuild_analysis::compiled_project::models::SplicedSql; /// Span rows. -#[inline] pub fn span_rows(spliced: &SplicedSql) -> Vec { - declaration_expansion::span_rows(spliced) + spliced + .spans + .iter() + .map(|span| SpanRow { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + }) + .collect() } diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/expanded.rs b/crates/sqlbuild-python-front/src/render/main/expansion/expanded.rs index f3493f3740..1981296de6 100644 --- a/crates/sqlbuild-python-front/src/render/main/expansion/expanded.rs +++ b/crates/sqlbuild-python-front/src/render/main/expansion/expanded.rs @@ -3,9 +3,7 @@ use crate::render::_helpers::expansion::macro_expansion; use crate::render::models::ExpansionOutput; use crate::render::models::NativeMacroExpansion; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Expand `sql` with the expansion's inputs, through the bridge's memo when one is active. #[inline] diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/expansion_result.rs b/crates/sqlbuild-python-front/src/render/main/expansion/expansion_result.rs index 3cc313db31..f32602167e 100644 --- a/crates/sqlbuild-python-front/src/render/main/expansion/expansion_result.rs +++ b/crates/sqlbuild-python-front/src/render/main/expansion/expansion_result.rs @@ -2,9 +2,7 @@ use crate::render::_helpers::expansion::macro_expansion_result; use crate::render::models::ExpansionRequest; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Expansion result. #[inline] diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/macro_expansion_classes.rs b/crates/sqlbuild-python-front/src/render/main/expansion/macro_expansion_classes.rs index ceebea1865..1aa3e08c4a 100644 --- a/crates/sqlbuild-python-front/src/render/main/expansion/macro_expansion_classes.rs +++ b/crates/sqlbuild-python-front/src/render/main/expansion/macro_expansion_classes.rs @@ -1,13 +1,13 @@ //! The classes, enums and helpers the native expansion builds facts and errors from, built once. -use crate::render::_helpers::expansion::macro_expansion_result; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::render::_helpers::expansion::macro_expansion_result::EXPANSION_CLASSES; +use crate::render::_helpers::expansion::macro_expansion_result::built_classes; +use pyo3::prelude::{Bound, PyResult, Python}; use pyo3::types::PyDict; /// The classes, enums and helpers the native expansion builds facts and errors from, built once. -#[inline] pub fn macro_expansion_classes(py: Python<'_>) -> PyResult> { - macro_expansion_result::macro_expansion_classes(py) + EXPANSION_CLASSES + .get_or_try_init(py, || built_classes(py).map(Bound::unbind)) + .map(|classes| classes.bind(py).clone()) } diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion.rs b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion.rs index 14a84995b5..a3fd9b002c 100644 --- a/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion.rs +++ b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion.rs @@ -1,12 +1,10 @@ //! Register the macro expansion functions and classes. -use crate::render::_helpers::expansion::macro_expansion; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::models::NativeMacroExpansion; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the macro expansion functions and classes. -#[inline] pub fn register_macro_expansion(module: &Bound<'_, PyModule>) -> PyResult<()> { - macro_expansion::register(module) + module.add_class::()?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion_result.rs b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion_result.rs index bd34acc97d..4807d9a877 100644 --- a/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion_result.rs +++ b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_expansion_result.rs @@ -1,12 +1,10 @@ //! Register the macro expansion result functions and classes. -use crate::render::_helpers::expansion::macro_expansion_result; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::_helpers::expansion::macro_expansion_result::expand_sql_macros; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the macro expansion result functions and classes. -#[inline] pub fn register_macro_expansion_result(module: &Bound<'_, PyModule>) -> PyResult<()> { - macro_expansion_result::register(module) + module.add_function(wrap_pyfunction!(expand_sql_macros, module)?) } diff --git a/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_modules.rs b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_modules.rs index abe97ff144..39cf08b062 100644 --- a/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_modules.rs +++ b/crates/sqlbuild-python-front/src/render/main/expansion/register_macro_modules.rs @@ -1,12 +1,13 @@ //! Register the macro modules functions and classes. -use crate::render::_helpers::expansion::macro_modules; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::_helpers::expansion::macro_modules::inventory_macro_modules; +use crate::render::_helpers::expansion::macro_modules::macro_module_load_plan; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the macro modules functions and classes. -#[inline] pub fn register_macro_modules(module: &Bound<'_, PyModule>) -> PyResult<()> { - macro_modules::register(module) + module.add_function(wrap_pyfunction!(inventory_macro_modules, module)?)?; + module.add_function(wrap_pyfunction!(macro_module_load_plan, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/render/main/sql_macros/register_sql_macro_inventory.rs b/crates/sqlbuild-python-front/src/render/main/sql_macros/register_sql_macro_inventory.rs index 2242424725..46d20a9dbb 100644 --- a/crates/sqlbuild-python-front/src/render/main/sql_macros/register_sql_macro_inventory.rs +++ b/crates/sqlbuild-python-front/src/render/main/sql_macros/register_sql_macro_inventory.rs @@ -1,12 +1,10 @@ //! Register the sql macro inventory functions and classes. -use crate::render::_helpers::sql_macros::sql_macro_inventory; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::render::_helpers::sql_macros::sql_macro_inventory::inventory_sql_macros; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the sql macro inventory functions and classes. -#[inline] pub fn register_sql_macro_inventory(module: &Bound<'_, PyModule>) -> PyResult<()> { - sql_macro_inventory::register(module) + module.add_function(wrap_pyfunction!(inventory_sql_macros, module)?) } diff --git a/crates/sqlbuild-python-front/src/render/main/sql_macros/with_sql_macro_mocks.rs b/crates/sqlbuild-python-front/src/render/main/sql_macros/with_sql_macro_mocks.rs index 951d255eb0..53b2a7076d 100644 --- a/crates/sqlbuild-python-front/src/render/main/sql_macros/with_sql_macro_mocks.rs +++ b/crates/sqlbuild-python-front/src/render/main/sql_macros/with_sql_macro_mocks.rs @@ -1,9 +1,7 @@ //! `macro_source_sql` with each call of a mocked SQL macro the model makes replaced by its mock. use crate::render::_helpers::sql_macros::sql_macro_mocks; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::types::PyDict; /// `macro_source_sql` with each call of a mocked SQL macro the model makes replaced by its mock. diff --git a/crates/sqlbuild-python-front/src/render/main/sql_macros/without_with_list_calls.rs b/crates/sqlbuild-python-front/src/render/main/sql_macros/without_with_list_calls.rs index 4f4d902195..b2743ce49c 100644 --- a/crates/sqlbuild-python-front/src/render/main/sql_macros/without_with_list_calls.rs +++ b/crates/sqlbuild-python-front/src/render/main/sql_macros/without_with_list_calls.rs @@ -1,9 +1,41 @@ //! The test's SQL with each `WITH`-list SQL macro call blanked, and the macros those calls name. -use crate::render::_helpers::sql_macros::sql_macro_tests; +use crate::render::_helpers::sql_macros::sql_macro_tests::ends_list; +use crate::render::_helpers::sql_macros::sql_macro_tests::is_blanked; +use crate::render::_helpers::sql_macros::sql_macro_tests::leading_space; +use sqlbuild_analysis::sql_macros::main::with_list_sql_macro_calls::with_list_sql_macro_calls; +use sqlbuild_analysis::sql_macros::models::SqlMacroCall; /// The test's SQL with each `WITH`-list SQL macro call blanked, and the macros those calls name. -#[inline] pub fn without_with_list_calls(sql: &str) -> (String, Vec) { - sql_macro_tests::without_with_list_calls(sql) + let calls: Vec = with_list_sql_macro_calls(sql); + if calls.is_empty() { + return (sql.to_owned(), Vec::new()); + } + let mut blanked: Vec<(usize, usize)> = Vec::new(); + for call in &calls { + blanked.push((call.start, call.end)); + let after: usize = call.end + leading_space(&sql[call.end..]); + if sql[after..].starts_with(',') { + blanked.push((after, after + 1)); + } else if ends_list(&sql[after..]) { + let before: &str = sql[..call.start].trim_end(); + if before.ends_with(',') { + blanked.push((before.len() - 1, before.len())); + } + } + } + let text: String = sql + .char_indices() + .map(|(offset, character)| { + let inside: bool = is_blanked(offset, &blanked); + if inside && !character.is_whitespace() { + ' ' + } else { + character + } + }) + .collect(); + let names: Vec = calls.into_iter().map(|call| call.name).collect(); + (text, names) } diff --git a/crates/sqlbuild-python-front/src/render/models.rs b/crates/sqlbuild-python-front/src/render/models.rs index a350fc00b4..4fc45f6c6e 100644 --- a/crates/sqlbuild-python-front/src/render/models.rs +++ b/crates/sqlbuild-python-front/src/render/models.rs @@ -4,10 +4,7 @@ use crate::render::_helpers::calls::macro_call_classes::CallClass; use crate::render::_helpers::calls::macro_call_classes::ClassInputs; use crate::render::_helpers::calls::macro_call_memo::MacroCallMemo; use crate::render::_helpers::calls::macro_call_memo::poisoned; -use crate::render::_helpers::calls::macro_call_rows::span_row; -use crate::render::_helpers::calls::macro_call_rows::span_tuple; use crate::render::_helpers::declarations::declaration_expansion::usage_visibility; -use crate::render::_helpers::declarations::declaration_values::constant_declaration_sql; use crate::render::_helpers::expansion::macro_evaluation::CallText; use crate::render::_helpers::expansion::macro_evaluation::Evaluation; use crate::render::_helpers::expansion::macro_evaluation::macro_argument_error; @@ -36,6 +33,9 @@ use crate::render::_helpers::expansion::macro_expansion_facts::ExpansionFacts; use crate::render::_helpers::expansion::macro_expansion_facts::ResourceRef; use crate::render::_helpers::expansion::macro_generated_references::GeneratedOrigin; use crate::render::_helpers::expansion::macro_generated_references::report_generated_references; +use crate::render::main::calls::span_row::span_row; +use crate::render::main::calls::span_tuple::span_tuple; +use crate::render::main::declarations::constant_declaration_sql::constant_declaration_sql; use crate::render::types::MacroScopes; use crate::render::types::SpanRow; use crate::render::types::StateTuple; diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_tolerant.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_tolerant.rs index 6f05103216..b6dc17d6cd 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_tolerant.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_tolerant.rs @@ -9,12 +9,9 @@ use sqlbuild_scopes::scope_index::models::{ }; use crate::scopes::_helpers::checks::scope_validation::replace; -use crate::scopes::_helpers::index::scope_objects::enum_text; -use crate::scopes::_helpers::index::scope_objects::field; -use crate::scopes::_helpers::index::scope_objects::optional_item; -use crate::scopes::_helpers::index::scope_objects::read_resource; -use crate::scopes::_helpers::index::scope_objects::root_object; -use crate::scopes::_helpers::index::scope_objects::tuple_attribute; +use crate::scopes::_helpers::index::scope_objects::{ + enum_text, field, optional_item, read_resource, root_object, tuple_attribute, +}; use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs index ab84f15f4d..c6fcb98f4c 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs @@ -1,48 +1,35 @@ //! Compile usage merging, placement validation and move relocation over a Python `ScopeIndex`. -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use crate::scopes::main::checks::relocated_declarations::relocated_declarations; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_scopes::compile_usages::models::UsageKey; use sqlbuild_scopes::placement::main::diagnostic_order::diagnostic_order; use sqlbuild_scopes::placement::main::placement_diagnostics::placement_diagnostics; -use sqlbuild_scopes::placement::main::relocate_for_resource_moves::relocate_for_resource_moves; use sqlbuild_scopes::placement::models::{ - DiagnosticOrderKey, PlacementCode, PlacementDiagnostic, PlacementIndex, Relocation, + DiagnosticOrderKey, PlacementCode, PlacementDiagnostic, PlacementIndex, }; use sqlbuild_scopes::scope_index::models::ResourceIdentity; -use crate::scopes::_helpers::declarations::placement::QualifiedRow; -use crate::scopes::_helpers::declarations::placement::resource_identity; -use crate::scopes::_helpers::index::scope_objects::enum_text; -use crate::scopes::_helpers::index::scope_objects::field; -use crate::scopes::_helpers::index::scope_objects::identity_row; -use crate::scopes::_helpers::index::scope_objects::optional_item; -use crate::scopes::_helpers::index::scope_objects::qualified_row; -use crate::scopes::_helpers::index::scope_objects::read_completeness; -use crate::scopes::_helpers::index::scope_objects::read_placement_index; -use crate::scopes::_helpers::index::scope_objects::root_object; -use crate::scopes::_helpers::index::scope_objects::tuple_attribute; -use crate::scopes::_helpers::index::scope_objects::value_of; -use crate::scopes::_helpers::relationships::compile_usages::MergeRows; -use crate::scopes::_helpers::relationships::compile_usages::SourceRow; -use crate::scopes::_helpers::relationships::compile_usages::merged_usage_rows; -use crate::scopes::_helpers::relationships::compile_usages::model_schema_scope_usages; +use crate::scopes::_helpers::declarations::placement::{QualifiedRow, resource_identity}; +use crate::scopes::_helpers::index::scope_objects::{ + enum_text, field, identity_row, optional_item, qualified_row, read_completeness, + read_placement_index, tuple_attribute, value_of, +}; +use crate::scopes::_helpers::relationships::compile_usages::{ + MergeRows, SourceRow, merged_usage_rows, +}; use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; -use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; -const RUNTIME_USAGE: usize = 2; +pub(crate) const RUNTIME_USAGE: usize = 2; /// Compile input groups whose items carry declaration usages, after the model inputs. -const COMPILE_USAGE_GROUPS: [&str; 5] = [ +pub(crate) const COMPILE_USAGE_GROUPS: [&str; 5] = [ "source_inputs", "sql_function_inputs", "audit_inputs", @@ -306,138 +293,6 @@ pub(crate) fn scope_relocated_declarations( Ok(relocated.map(|(records, _)| records.unbind())) } -/// Relocated records after every move in `moves`, and per record whether it lacks a needed group. -pub(crate) fn relocated_declarations<'py>( - index: &Bound<'py, PyAny>, - moves: &[(ResourceIdentity, String)], -) -> PyResult, Vec)>> { - let py: Python<'py> = index.py(); - if !read_completeness(index)?[RUNTIME_USAGE] { - return Ok(None); - } - let placement: PlacementIndex = read_placement_index(index)?; - let relocations: Option> = py - .compiler_detach(|| Ok(relocate_for_resource_moves(&placement, moves))) - .map_err(compiler_error)?; - let Some(relocations) = relocations else { - return Ok(None); - }; - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - let records: Bound<'_, PyTuple> = tuple_attribute(index, "declarations")?; - let mut moved: Vec> = Vec::with_capacity(relocations.len()); - let mut grouping: Vec = Vec::with_capacity(relocations.len()); - for item in relocations { - let owning_path: Bound<'_, PyAny> = item.owning_path.into_bound_py_any(py)?; - let record: Bound<'py, PyAny> = replace( - &records.get_item(item.declaration)?, - &[ - ("path", item.path.into_bound_py_any(py)?), - ( - "scope", - classes.member(RecordId::ScopeKind, item.scope.as_str())?, - ), - ( - "ownership_root", - root_object(&classes, &item.ownership_root)?, - ), - ("owning_path", owning_path), - ], - )?; - moved.push(record); - grouping.push(item.requires_group); - } - Ok(Some((PyTuple::new(py, moved)?, grouping))) -} - -fn usage_records<'py>(records: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn usage_records<'py>(records: &Bound<'py, PyAny>) -> PyResult>> { records.try_iter()?.collect() } - -/// Compile inputs' canonical index with all usage and placement facts. -pub(crate) fn scope_index_with_compile_usages<'py>( - inputs: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = inputs.py(); - let scope: Bound<'py, PyAny> = native_field(inputs, "declaration_scope")?; - let resolver: Bound<'py, PyAny> = if scope.is_none() { - scope - } else { - native_field(&scope, "resolver")? - }; - let schema_files: Bound<'py, PyAny> = native_field( - &native_field(inputs, "discovered_inputs")?, - "model_schema_files", - )?; - let mut usages: Vec> = Vec::new(); - usages.extend(usage_records( - model_schema_scope_usages(&schema_files, &resolver)?.as_any(), - )?); - let digests: bool = sqlbuild_python_base::compile_context::main::delta::capturing::capturing(); - for model in native_field(inputs, "model_inputs")?.try_iter()? { - let model: Bound<'py, PyAny> = model?; - let macro_usages: Vec> = - usage_records(&native_field(&model, "macro_usages")?)?; - let declaration_usages: Vec> = - usage_records(&native_field(&model, "declaration_usages")?)?; - if digests { - sqlbuild_python_base::compile_context::main::image_facts::record_model_usages::record_model_usages( - &model, - (¯o_usages, &declaration_usages), - )?; - } - usages.extend(macro_usages); - usages.extend(declaration_usages); - } - for group in COMPILE_USAGE_GROUPS { - for item in native_field(inputs, group)?.try_iter()? { - usages.extend(usage_records(&native_field(&item?, "declaration_usages")?)?); - } - } - let enforce_placement: bool = native_field( - &native_field(&native_field(inputs, "project_config")?, "scopes")?, - "enforce_placement", - )? - .extract()?; - Ok(scope_placement_validated_index( - &native_field(inputs, "scope_index")?, - enforce_placement, - Some(PyTuple::new(py, usages)?), - )? - .into_bound(py)) -} - -/// Python's `validate_scope_index`: raise one `ScopeValidationError` for its error diagnostics. -#[pyfunction] -#[pyo3(signature = (*, index))] -pub(crate) fn validate_scope_index(index: &Bound<'_, PyAny>) -> Result<(), PyErr> { - let py: Python<'_> = index.py(); - let error: Bound<'_, PyAny> = enum_member(py, RecordId::ScopeDiagnosticSeverity, "ERROR")?; - let mut errors: Vec> = Vec::new(); - for item in native_field(index, "diagnostics")?.try_iter()? { - let item: Bound<'_, PyAny> = item?; - if native_field(&item, "severity")?.is(&error) { - errors.push(item); - } - } - if errors.is_empty() { - return Ok(()); - } - let mut message: String = String::from("Invalid declaration scope index:"); - for item in &errors { - let code: String = native_field(&native_field(item, "code")?, "value")?.extract()?; - let text: String = native_field(item, "message")?.extract()?; - message.push_str(&format!("\n[{code}] {text}")); - } - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("diagnostics", PyTuple::new(py, errors)?)?; - Err(PyErr::from_value( - error_class(py, ErrorId::ScopeValidation)?.call((message,), Some(&options))?, - )) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(validate_scope_index, module)?)?; - module.add_function(wrap_pyfunction!(scope_placement_validated_index, module)?)?; - module.add_function(wrap_pyfunction!(scope_relocated_declarations, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_visibility.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_visibility.rs index ba33e6ec00..024b9d0001 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_visibility.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_visibility.rs @@ -1,23 +1,20 @@ //! Static declaration visibility of scope targets and authored paths over a `ScopeLookup`. -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; use sqlbuild_scopes::scope_index::models::{ OwnershipRoot, OwnershipRootKind, RejectedPath, ResourceIdentity, ResourceKind, ResourceRecord, }; -use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; -use crate::scopes::_helpers::index::scope_lookup::ResolvedRows; -use crate::scopes::_helpers::index::scope_lookup::TargetRows; -use crate::scopes::_helpers::index::scope_lookup::scope_path_error; -use crate::scopes::_helpers::index::scope_objects::qualified_row; -use crate::scopes::_helpers::index::scope_objects::read_resource; -use crate::scopes::_helpers::index::scope_objects::tuple_attribute; +use crate::scopes::_helpers::index::scope_lookup::{ + NativeScopeLookup, ResolvedRows, TargetRows, scope_path_error, +}; +use crate::scopes::_helpers::index::scope_objects::{ + qualified_row, read_resource, tuple_attribute, +}; use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -101,7 +98,7 @@ fn target_query<'py>( /// `query_target`: the `ScopeTargetQuery` of `target`; invalid text raises the scope error. #[pyfunction] #[pyo3(name = "query_scope_target", signature = (*, lookup, target))] -fn scope_query_target( +pub(crate) fn scope_query_target( py: Python<'_>, lookup: &Bound<'_, PyAny>, target: &Bound<'_, PyAny>, @@ -124,7 +121,7 @@ fn resolved( /// Python's `resolve_scope_visibility`: visible facts and inaccessible reasons per target. #[pyfunction] #[pyo3(signature = (*, lookup, target))] -fn resolve_scope_visibility( +pub(crate) fn resolve_scope_visibility( py: Python<'_>, lookup: &Bound<'_, PyAny>, target: &Bound<'_, PyAny>, @@ -135,7 +132,7 @@ fn resolve_scope_visibility( /// Python's `resolve_scope_declaration_visibility`: visible facts and inaccessible identities. #[pyfunction] #[pyo3(signature = (*, lookup, target))] -fn resolve_scope_declaration_visibility( +pub(crate) fn resolve_scope_declaration_visibility( py: Python<'_>, lookup: &Bound<'_, PyAny>, target: &Bound<'_, PyAny>, @@ -208,7 +205,7 @@ fn resolved_visibility( /// Visible and inaccessible declarations for an authored path, including declaration files. #[pyfunction] #[pyo3(name = "resolve_scope_path_visibility", signature = (*, lookup, path))] -fn scope_resolve_path_visibility( +pub(crate) fn scope_resolve_path_visibility( py: Python<'_>, lookup: &Bound<'_, PyAny>, path: &Bound<'_, PyAny>, @@ -261,14 +258,3 @@ fn scope_resolve_path_visibility( )?, )) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scope_query_target, module)?)?; - module.add_function(wrap_pyfunction!(resolve_scope_visibility, module)?)?; - module.add_function(wrap_pyfunction!( - resolve_scope_declaration_visibility, - module - )?)?; - module.add_function(wrap_pyfunction!(scope_resolve_path_visibility, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_contexts.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_contexts.rs index fafbf520b4..b726c94ae8 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_contexts.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_contexts.rs @@ -1,10 +1,9 @@ //! Declaration resolution contexts built natively from a Python scope lookup's positions. -use crate::scopes::models::NativeDeclarationContexts; use std::collections::HashMap; use pyo3::FromPyObject; -use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, Py, PyAny}; use pyo3::types::PyDict; use sqlbuild_scopes::scope_index::models::VisibleEntry; @@ -59,8 +58,3 @@ pub(crate) struct ContextParts<'py> { pub(crate) inaccessible: [Bound<'py, PyDict>; 3], pub(crate) visibility: [Bound<'py, PyDict>; 3], } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs index 83357b83b3..cae5e29d27 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs @@ -1,31 +1,19 @@ //! The persistent scope index cache: fingerprint, verified read and atomic write. +use crate::scopes::main::index::project_fingerprint::project_fingerprint; use std::path::PathBuf; -use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_scopes::scope_cache::main::encode_scope_cache::encode_scope_cache; -use sqlbuild_scopes::scope_cache::main::project_fingerprint_inputs::project_fingerprint_inputs; -use sqlbuild_scopes::scope_cache::main::read_scope_cache::read_scope_cache; -use sqlbuild_scopes::scope_cache::main::scope_fingerprint::scope_fingerprint; -use sqlbuild_scopes::scope_cache::main::write_scope_cache::write_scope_cache; -use sqlbuild_scopes::scope_cache::models::{CachedIndex, Fingerprint, FingerprintInputs}; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyResult, Python}; +use pyo3::pyfunction; -use crate::scopes::_helpers::index::scope_objects::index_object; -use crate::scopes::_helpers::index::scope_objects::read_completeness; -use crate::scopes::_helpers::index::scope_objects::read_index; -use crate::scopes::models::ScopeClasses; -use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::types::CompilerDetach; const METADATA_MODULE: &str = "importlib.metadata"; const DISTRIBUTION_NAME: &str = "sqlbuild"; const UNKNOWN_VERSION: &str = "0+unknown"; /// `importlib.metadata.version("sqlbuild")`, or `0+unknown` when it is not installed. -fn sqlbuild_version(py: Python<'_>) -> PyResult { +pub(crate) fn sqlbuild_version(py: Python<'_>) -> PyResult { let metadata: Bound<'_, PyModule> = py.import(METADATA_MODULE)?; match native_field(&metadata, "version")?.call1((DISTRIBUTION_NAME,)) { Ok(version) => version.extract(), @@ -36,72 +24,12 @@ fn sqlbuild_version(py: Python<'_>) -> PyResult { } } -/// The scope index fingerprint of `project_dir`, `None` when it cannot be read. -pub(crate) fn project_fingerprint( - py: Python<'_>, - project_dir: PathBuf, -) -> PyResult> { - let inputs: FingerprintInputs = project_fingerprint_inputs(sqlbuild_version(py)?); - let fingerprint: Fingerprint = py - .compiler_detach(|| Ok(scope_fingerprint(&project_dir, &inputs))) - .map_err(compiler_error)?; - match fingerprint { - Fingerprint::Hash(hash) => Ok(Some(hash)), - Fingerprint::Unavailable => Ok(None), - Fingerprint::Unserializable => Err(PyTypeError::new_err( - "a scope-relevant config value is not JSON serializable", - )), - } -} - /// Hash the canonical scope-relevant authored inputs, or `None` when they cannot be read. #[pyfunction] #[pyo3(signature = (*, project_dir))] -fn scope_index_fingerprint(py: Python<'_>, project_dir: PathBuf) -> PyResult> { - project_fingerprint(py, project_dir) -} - -/// The verified cached `ScopeIndex` at `path` for `fingerprint`; every fault is a miss. -pub(crate) fn read_scope_index_cache( - py: Python<'_>, - path: PathBuf, - fingerprint: String, -) -> PyResult>> { - let cached = py - .compiler_detach(|| Ok(read_scope_cache(&path, &fingerprint))) - .map_err(compiler_error)?; - let Ok(index) = cached else { - return Ok(None); - }; - match index_object(&ScopeClasses::new(py)?, &index) { - Ok(object) => Ok(Some(object.unbind())), - Err(error) if error.is_instance_of::(py) => Ok(None), - Err(error) => Err(error), - } -} - -/// Atomically write one complete `ScopeIndex` for `fingerprint`; whether the write landed. -pub(crate) fn write_scope_index_cache( +pub(crate) fn scope_index_fingerprint( py: Python<'_>, - location: (PathBuf, String), - fingerprint: String, - index: &Bound<'_, PyAny>, -) -> PyResult { - if !read_completeness(index)?.iter().all(|done| *done) { - return Ok(false); - } - let index: CachedIndex = read_index(index)?; - let (directory, file_name) = location; - py.compiler_detach(|| { - Ok(match encode_scope_cache(&index, &fingerprint) { - Ok(envelope) => matches!(write_scope_cache(&directory, &file_name, &envelope), Ok(())), - Err(_) => false, - }) - }) - .map_err(compiler_error) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scope_index_fingerprint, module)?)?; - Ok(()) + project_dir: PathBuf, +) -> PyResult> { + project_fingerprint(py, project_dir) } diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs index 1716405bb2..c771639a5b 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs @@ -1,41 +1,32 @@ //! The static scope index built natively from discovered inputs, as the Python scope models. +use crate::scopes::main::index::open_native_scope_index::open_native_scope_index; use crate::scopes::models::NativeScopeIndex; use std::collections::HashMap; use pyo3::exceptions::PyKeyError; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; -use sqlbuild_scopes::scope_index::main::build_scope_index::build_scope_index; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_scopes::scope_index::main::macro_source_digest::macro_source_digest; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; use sqlbuild_scopes::scope_index::models::{ DeclarationIdentity, DeclarationInput, DeclarationKind, RejectedPath, ResourceIdentity, - ResourceInput, ResourceKind, ResourceRoot, ScopeDeferral, ScopeIndex, ScopeInputs, ScopeKind, + ResourceInput, ResourceKind, ResourceRoot, ScopeIndex, ScopeInputs, ScopeKind, }; -use crate::discovery::models::ModelFileTexts; -use crate::discovery::models::NativeModelFile; +use crate::discovery::models::{ModelFileTexts, NativeModelFile}; use crate::scopes::_helpers::index::scope_lookup::scope_path_error; -use crate::scopes::_helpers::index::scope_objects::field; -use crate::scopes::_helpers::index::scope_objects::optional_item; -use crate::scopes::_helpers::index::scope_objects::root_object; -use crate::scopes::_helpers::index::scope_objects::value_of; -use crate::scopes::_helpers::relationships::relationship_scan::ScenarioRow; -use crate::scopes::_helpers::relationships::relationship_scan::TestFileRow; +use crate::scopes::_helpers::index::scope_objects::{field, optional_item, root_object, value_of}; +use crate::scopes::_helpers::relationships::relationship_scan::{ScenarioRow, TestFileRow}; use crate::scopes::models::ScopeClasses; -use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::records::main::fields::path_stem::path_stem; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_base::types::CompilerDetach; /// Why `UnindexableScopeError` is raised; discovery only yields normalized project paths. const UNINDEXABLE_SCOPE_MESSAGE: &str = "Native scope indexing could not place a discovered path or resource kind. Discovery always yields normalized project-relative paths, so this is a SQLBuild bug; please file an issue with the project layout"; @@ -75,15 +66,15 @@ struct Placement { } /// The builder's walk of discovered inputs: native inputs and the Python values beside them. -struct Walk<'py> { +pub(crate) struct Walk<'py> { classes: ScopeClasses<'py>, - inputs: ScopeInputs, + pub(crate) inputs: ScopeInputs, resource_identities: Vec>, /// Each resource's kind and root, which pick its ownership root. resource_roots: Vec<(ResourceKind, ResourceRoot)>, declarations: Vec>, /// Whether every walked kind, scope and root parsed natively. - indexable: bool, + pub(crate) indexable: bool, } fn text(value: &Bound<'_, PyAny>) -> PyResult { @@ -308,7 +299,7 @@ impl<'py> Walk<'py> { )) } - fn add_models(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { + pub(crate) fn add_models(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { for model_file in items(inputs, "model_files")? { let header: Bound<'py, PyAny> = header_name(&model_file)?; let (model_name, path, folder): (String, String, String) = @@ -390,7 +381,7 @@ impl<'py> Walk<'py> { ) } - fn add_resources(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { + pub(crate) fn add_resources(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { for test_file in items(inputs, "test_files")? { let path: Bound<'py, PyAny> = native_field(&test_file, "relative_path")?; for block in items(&test_file, "blocks")? { @@ -458,7 +449,7 @@ impl<'py> Walk<'py> { }) } - fn add_declaration_files(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { + pub(crate) fn add_declaration_files(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { for enum_file in items(inputs, "enum_files")? { for declaration in items(&enum_file, "declarations")? { let identity: Bound<'py, PyAny> = self.declaration_identity( @@ -512,7 +503,7 @@ impl<'py> Walk<'py> { self.add_declaration(values, placement) } - fn add_named_declarations(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { + pub(crate) fn add_named_declarations(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { let audit_kind: String = DeclarationKind::Audit.as_str().to_owned(); for audit_file in items(inputs, "audit_files")? { let stem_name: Bound<'py, PyAny> = @@ -621,7 +612,7 @@ impl<'py> Walk<'py> { Ok((line, metadata)) } - fn add_macros( + pub(crate) fn add_macros( &mut self, inputs: &Bound<'py, PyAny>, macros: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -701,7 +692,7 @@ fn resource_root<'py>( )) } -fn materialise<'py>(walk: Walk<'py>, native: ScopeIndex) -> PyResult { +pub(crate) fn materialise<'py>(walk: Walk<'py>, native: ScopeIndex) -> PyResult { let classes: &ScopeClasses<'py> = &walk.classes; let py: Python<'py> = classes.py(); let mut resources: Vec> = Vec::with_capacity(native.resources.len()); @@ -853,46 +844,13 @@ pub(crate) fn scenario_row(scenario: &Bound<'_, PyAny>) -> PyResult )) } -/// Python's `open_native_scope_index`: the natively built index of discovered inputs. -#[pyfunction] -#[pyo3(signature = (*, discovered_inputs, loaded_macros, static_macros = None))] -pub(crate) fn open_native_scope_index( - py: Python<'_>, - discovered_inputs: &Bound<'_, PyAny>, - loaded_macros: &Bound<'_, PyAny>, - static_macros: Option>, -) -> PyResult { - let static_macros: Bound<'_, PyAny> = match static_macros { - Some(static_macros) => static_macros, - None => PyTuple::empty(py).into_any(), - }; - let static_macros: &Bound<'_, PyAny> = &static_macros; - let mut walk: Walk<'_> = Walk::new(py)?; - walk.add_models(discovered_inputs)?; - walk.add_resources(discovered_inputs)?; - walk.add_declaration_files(discovered_inputs)?; - walk.add_named_declarations(discovered_inputs)?; - walk.add_macros(discovered_inputs, (loaded_macros, static_macros))?; - if !walk.indexable { - return Err(unindexable(py)?); - } - let inputs: ScopeInputs = std::mem::take(&mut walk.inputs); - let built: Result = py - .compiler_detach(|| Ok(build_scope_index(inputs))) - .map_err(compiler_error)?; - let Ok(native) = built else { - return Err(unindexable(py)?); - }; - materialise(walk, native) -} - /// Python's `build_scope_index`: the natively built, partially valid index of discovered inputs. #[pyfunction] #[pyo3( name = "build_scope_index", signature = (*, discovered_inputs, loaded_macros = None, static_macros = None) )] -fn build_scope_index_binding( +pub(crate) fn build_scope_index_binding( py: Python<'_>, discovered_inputs: &Bound<'_, PyAny>, loaded_macros: Option>, @@ -906,10 +864,3 @@ fn build_scope_index_binding( open_native_scope_index(py, discovered_inputs, &loaded_macros, static_macros)?; Ok(native.index.clone_ref(py)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(open_native_scope_index, module)?)?; - module.add_function(wrap_pyfunction!(build_scope_index_binding, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs index e0ea677f39..519c22847b 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs @@ -1,14 +1,11 @@ //! The scope lookup of a full scope index: canonical orders, mappings, targets and visibility. -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; -use pyo3::{pyclass, pyfunction, wrap_pyfunction}; +use pyo3::{pyclass, pyfunction}; use sqlbuild_scopes::scope_index::main::identity_failure_message::identity_failure_message; use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; -use sqlbuild_scopes::scope_index::main::report_lookup::report_lookup; use sqlbuild_scopes::scope_index::main::report_target::report_target; use sqlbuild_scopes::scope_index::main::resolve_report_visibility::resolve_report_visibility; use sqlbuild_scopes::scope_index::models::{ @@ -16,13 +13,9 @@ use sqlbuild_scopes::scope_index::models::{ ResolvedVisibility, ResolvedVisible, ResourceRecord, TargetFailure, TargetMatch, VisibilityGroups, }; -use sqlbuild_scopes::scope_report::main::cached_report_records::cached_report_records; -use crate::scopes::_helpers::declarations::placement::QualifiedRow; -use crate::scopes::_helpers::declarations::placement::qualified_identity; -use crate::scopes::_helpers::index::scope_objects::read_index; +use crate::scopes::_helpers::declarations::placement::{QualifiedRow, qualified_identity}; use crate::scopes::_helpers::index::scope_objects::resource_identity_object; -use crate::scopes::_helpers::index::scope_objects::tuple_attribute; use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -53,17 +46,17 @@ pub(crate) struct NativeScopeLookup { pub(crate) records: ReportRecords, pub(crate) lookup: ReportLookup, /// Each input resource, declaration and grant's canonical position. - canonical: CanonicalPositions, + pub(crate) canonical: CanonicalPositions, } /// The inverse of the lookup's canonical orders. -struct CanonicalPositions { - resources: Vec, - declarations: Vec, - grants: Vec, +pub(crate) struct CanonicalPositions { + pub(crate) resources: Vec, + pub(crate) declarations: Vec, + pub(crate) grants: Vec, } -fn inverse(order: &[usize]) -> Vec { +pub(crate) fn inverse(order: &[usize]) -> Vec { let mut positions: Vec = vec![0; order.len()]; for (canonical, input) in order.iter().enumerate() { positions[*input] = canonical; @@ -82,7 +75,7 @@ pub(crate) fn grant_kind(value: &str) -> PyResult { } /// Input positions as canonical positions. -fn canonical(order: &[usize], positions: Vec) -> Vec { +pub(crate) fn canonical(order: &[usize], positions: Vec) -> Vec { positions .into_iter() .map(|position| order[position]) @@ -171,7 +164,10 @@ fn resolved_rows(resolved: ResolvedVisibility, grants: &[usize]) -> ResolvedRows ) } -fn picked<'py>(values: &Bound<'py, PyTuple>, order: &[usize]) -> PyResult> { +pub(crate) fn picked<'py>( + values: &Bound<'py, PyTuple>, + order: &[usize], +) -> PyResult> { let items: Vec> = order .iter() .map(|position| values.get_item(*position)) @@ -188,7 +184,7 @@ fn mapping_proxy<'py>(entries: &Bound<'py, PyDict>) -> PyResult( +pub(crate) fn grouped<'py>( groups: &KeyedGroups, records: &Bound<'py, PyTuple>, key: &str, @@ -201,7 +197,7 @@ fn grouped<'py>( mapping_proxy(&entries) } -fn path_grouped<'py>( +pub(crate) fn path_grouped<'py>( groups: &KeyedGroups, records: &Bound<'py, PyTuple>, paths: &[String], @@ -224,7 +220,7 @@ fn positions_by<'py>( mapping_proxy(&mapping) } -fn visibility_index<'py>( +pub(crate) fn visibility_index<'py>( classes: &ScopeClasses<'py>, canonical_declarations: &Bound<'py, PyTuple>, visibility: &VisibilityGroups, @@ -256,91 +252,10 @@ pub(crate) fn scope_path_error(py: Python<'_>, message: String) -> PyResult, index: &Bound<'_, PyAny>) -> PyResult> { - let records: ReportRecords = cached_report_records(&read_index(index)?); - let built = py - .compiler_detach(|| Ok(report_lookup(&records))) - .map_err(compiler_error)?; - let lookup: ReportLookup = match built { - Ok(lookup) => lookup, - Err(rejected) => return Err(scope_path_error(py, rejected_path_message(&rejected))?), - }; - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - let resources: Bound<'_, PyTuple> = tuple_attribute(index, "resources")?; - let declarations: Bound<'_, PyTuple> = tuple_attribute(index, "declarations")?; - let usages: Bound<'_, PyTuple> = tuple_attribute(index, "usages")?; - let grants: Bound<'_, PyTuple> = tuple_attribute(index, "grants")?; - let canonical_declarations: Bound<'_, PyTuple> = picked(&declarations, &lookup.declarations)?; - let canonical_index: Bound<'_, PyAny> = classes.model(RecordId::ScopeIndex)?.call1(( - picked(&tuple_attribute(index, "ownership_roots")?, &lookup.roots)?, - picked(&resources, &lookup.resources)?, - &canonical_declarations, - picked(&usages, &lookup.usages)?, - picked(&grants, &lookup.grants)?, - native_field(index, "visibility")?, - native_field(index, "inaccessible")?, - picked(&tuple_attribute(index, "diagnostics")?, &lookup.diagnostics)?, - native_field(index, "completeness")?, - ))?; - let groups = &lookup.groups; - let mappings: [Bound<'_, PyAny>; 6] = [ - grouped(&groups.resources, &resources, "identity")?, - path_grouped( - &groups.resources_by_path, - &resources, - &lookup.resource_paths, - )?, - grouped(&groups.declarations, &declarations, "identity")?, - grouped(&groups.usages_by_consumer, &usages, "consumer")?, - grouped(&groups.usages_by_declaration, &usages, "declaration")?, - grouped(&groups.grants_by_resource, &grants, "resource")?, - ]; - let visibility: Bound<'_, PyAny> = - visibility_index(&classes, &canonical_declarations, &lookup.visibility)?; - let canonical: CanonicalPositions = CanonicalPositions { - resources: inverse(&lookup.resources), - declarations: inverse(&lookup.declarations), - grants: inverse(&lookup.grants), - }; - let native: Py = Py::new( - py, - NativeScopeLookup { - records, - lookup, - canonical, - }, - )?; - let [ - by_identity, - by_path, - by_declaration, - by_consumer, - by_used, - by_resource, - ] = mappings; - Ok(classes - .model(RecordId::ScopeLookup)? - .call1(( - canonical_index, - by_identity, - by_path, - by_declaration, - by_consumer, - by_used, - by_resource, - visibility, - native, - ))? - .unbind()) -} - /// The canonical project-relative POSIX path; absolute or escaping paths raise. #[pyfunction] #[pyo3(name = "normalize_scope_path", signature = (*, path))] -fn scope_normalize_path(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn scope_normalize_path(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult { let path: String = path.str()?.extract()?; match normalize_scope_path(&path) { Ok(normalized) => Ok(normalized), @@ -350,10 +265,3 @@ fn scope_normalize_path(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(build_scope_lookup, module)?)?; - module.add_function(wrap_pyfunction!(scope_normalize_path, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs index 016b1beb3e..993733662e 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs @@ -13,15 +13,10 @@ use sqlbuild_scopes::scope_index::models::{ ResourceKind, ResourceRecord, }; -use crate::scopes::_helpers::declarations::placement::DeclarationRow; -use crate::scopes::_helpers::declarations::placement::IdentityRow; -use crate::scopes::_helpers::declarations::placement::QualifiedRow; -use crate::scopes::_helpers::declarations::placement::RootRow; -use crate::scopes::_helpers::declarations::placement::declaration_identity; -use crate::scopes::_helpers::declarations::placement::declaration_record; -use crate::scopes::_helpers::declarations::placement::ownership_root; -use crate::scopes::_helpers::declarations::placement::qualified_identity; -use crate::scopes::_helpers::declarations::placement::resource_identity; +use crate::scopes::_helpers::declarations::placement::{ + DeclarationRow, IdentityRow, QualifiedRow, RootRow, declaration_identity, declaration_record, + ownership_root, qualified_identity, resource_identity, +}; use crate::scopes::_helpers::index::scope_lookup::grant_kind; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs index b84011d0f1..4d5ea1010e 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs @@ -3,11 +3,9 @@ use std::collections::HashMap; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_scopes::compile_usages::main::merged_usages::merged_usages; use sqlbuild_scopes::compile_usages::main::schema_scope_usages::schema_scope_usages; use sqlbuild_scopes::compile_usages::models::{ @@ -276,8 +274,3 @@ pub(crate) fn merged_usage_rows( combined, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(model_schema_scope_usages, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_names.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_names.rs index eb801aa90f..ae27ddc070 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_names.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_names.rs @@ -1,7 +1,7 @@ //! Expected-model relationship names and top-level CTEs of SQL tests and scenarios, scanned natively. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_scopes::relationship_names::main::expected_model_names::expected_model_names; use sqlbuild_scopes::relationship_names::main::top_level_ctes::scan_top_level_ctes; use sqlbuild_scopes::relationship_names::models::{ @@ -26,7 +26,7 @@ fn source(scenario: bool) -> RelationshipSource { /// Each `(sql, file label)` body's expected models or the scan's error. #[pyfunction] -fn scope_expected_model_names( +pub(crate) fn scope_expected_model_names( py: Python<'_>, texts: Vec<(String, String)>, scenario: bool, @@ -47,7 +47,7 @@ fn scope_expected_model_names( /// Each `(sql, file label)` test body's top-level CTEs or the scan's error. #[pyfunction] -fn scope_test_ctes( +pub(crate) fn scope_test_ctes( py: Python<'_>, texts: Vec<(String, String)>, syntax: LexicalSyntaxInput, @@ -71,9 +71,3 @@ fn scope_test_ctes( }) .collect()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scope_expected_model_names, module)?)?; - module.add_function(wrap_pyfunction!(scope_test_ctes, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs index 396b548083..35dc69385e 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs @@ -1,5 +1,6 @@ //! Scope report requests read from, and results built as, the Python report models. +use crate::scopes::main::reports::json_to_python::json_to_python; use pyo3::IntoPyObjectExt; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; @@ -12,14 +13,12 @@ use sqlbuild_scopes::scope_report::main::cached_diagnostic_facts::cached_diagnos use sqlbuild_scopes::scope_report::models::{ BrowseResult, DeclarationReport, DeclarationTarget, Explanation, ListResult, MovePreview, Provenance, QueryTarget, ReportDiagnostic, ReportFilters, ResourceReport, ScopeChainEntry, - ScopeFolder, ScopeReport, ScopeResult, ScopeSection, SourceLocation, + ScopeFolder, ScopeReport, ScopeSection, SourceLocation, }; -use crate::scopes::_helpers::index::scope_objects::field; -use crate::scopes::_helpers::index::scope_objects::optional_item; -use crate::scopes::_helpers::index::scope_objects::read_declaration_identity; -use crate::scopes::_helpers::index::scope_objects::read_diagnostic; -use crate::scopes::_helpers::index::scope_objects::read_qualified; +use crate::scopes::_helpers::index::scope_objects::{ + field, optional_item, read_declaration_identity, read_diagnostic, read_qualified, +}; use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; @@ -98,43 +97,6 @@ pub(crate) fn read_filters(filters: &Bound<'_, PyAny>) -> PyResult, value: JsonValue, tuples: bool) -> PyResult { - match value { - JsonValue::Null => Ok(py.None()), - JsonValue::Bool(flag) => flag.into_py_any(py), - JsonValue::Integer(integer) => match integer.to_i128() { - Some(number) => number.into_py_any(py), - None => Ok(py - .import("builtins")? - .getattr("int")? - .call1((integer.as_str(),))? - .unbind()), - }, - JsonValue::Float(number) => number.into_py_any(py), - JsonValue::String(text) => text.into_py_any(py), - JsonValue::Utf16(units) => String::from_utf16_lossy(&units).into_py_any(py), - JsonValue::Array(items) => { - let values: Vec = items - .into_iter() - .map(|item| json_to_python(py, item, tuples)) - .collect::>()?; - if tuples { - PyTuple::new(py, values)?.into_py_any(py) - } else { - PyList::new(py, values)?.into_py_any(py) - } - } - JsonValue::Object(entries) => { - let dictionary: Bound<'_, PyDict> = PyDict::new(py); - for (key, item) in entries { - dictionary.set_item(key, json_to_python(py, item, tuples)?)?; - } - dictionary.into_py_any(py) - } - } -} - /// `plain_value(int(integer))`: a 64-bit signed or unsigned number, else the wide-integer error. fn plain_integer(integer: &JsonInteger) -> PyResult { let Some(number) = integer.to_i128() else { @@ -504,7 +466,7 @@ fn read_declaration_report(report: &Bound<'_, PyAny>) -> PyResult, name: &str, ) -> PyResult> { @@ -514,7 +476,7 @@ fn read_declaration_reports( .collect() } -fn read_section(section: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn read_section(section: &Bound<'_, PyAny>) -> PyResult { Ok(ScopeSection { name: field(section, "name")?, total: field(section, "total")?, @@ -529,7 +491,7 @@ fn read_section(section: &Bound<'_, PyAny>) -> PyResult { }) } -fn read_folder(folder: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn read_folder(folder: &Bound<'_, PyAny>) -> PyResult { let mut kind_counts: Vec<(String, i64)> = Vec::new(); for entry in native_field(folder, "kind_counts")?.try_iter()? { let (kind, count): (Bound<'_, PyAny>, i64) = entry?.extract()?; @@ -552,7 +514,7 @@ fn read_folder(folder: &Bound<'_, PyAny>) -> PyResult { }) } -fn read_diagnostics(item: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn read_diagnostics(item: &Bound<'_, PyAny>) -> PyResult> { native_field(item, "diagnostics")? .try_iter()? .map(|diagnostic| { @@ -587,7 +549,7 @@ fn read_move(preview: &Bound<'_, PyAny>) -> PyResult { }) } -fn read_report(report: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn read_report(report: &Bound<'_, PyAny>) -> PyResult { let resource: Bound<'_, PyAny> = native_field(report, "resource")?; Ok(ScopeReport { resource: ResourceReport { @@ -628,30 +590,3 @@ fn read_report(report: &Bound<'_, PyAny>) -> PyResult { complete: field(report, "complete")?, }) } - -/// A `ScopeReport`, `ScopeBrowseResult` or, otherwise, `ScopeListResult` as its native result. -pub(crate) fn read_result( - classes: &ScopeClasses<'_>, - result: &Bound<'_, PyAny>, -) -> PyResult { - if result.is_instance(&classes.model(RecordId::ScopeReport)?)? { - return Ok(ScopeResult::Report(Box::new(read_report(result)?))); - } - if result.is_instance(&classes.model(RecordId::ScopeBrowseResult)?)? { - return Ok(ScopeResult::Browse(BrowseResult { - folder: field(result, "folder")?, - folders: native_field(result, "folders")? - .try_iter()? - .map(|folder| read_folder(&folder?)) - .collect::>()?, - diagnostics: read_diagnostics(result)?, - complete: field(result, "complete")?, - })); - } - Ok(ScopeResult::List(ListResult { - folder: field(result, "folder")?, - declarations: read_declaration_reports(result, "declarations")?, - section: read_section(&native_field(result, "section")?)?, - diagnostics: read_diagnostics(result)?, - })) -} diff --git a/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs b/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs index cfb49a189b..1494f26d96 100644 --- a/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs +++ b/crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs @@ -1,70 +1,54 @@ //! `sqb scope` reports over a `ScopeLookup`, their serialization and the metadata projection. +use crate::scopes::main::reports::json_to_python::json_to_python; +use crate::scopes::main::reports::read_result::read_result; +use crate::scopes::main::reports::scope_browse::scope_browse; +use crate::scopes::main::reports::scope_explain::scope_explain; +use crate::scopes::main::reports::scope_list::scope_list; +use crate::scopes::main::reports::scope_report::scope_report; use crate::scopes::models::ExplainArguments; use crate::scopes::models::ListArguments; use crate::scopes::models::ReportArguments; use pyo3::exceptions::{PyIndexError, PyValueError}; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::json::models::JsonValue; use sqlbuild_scopes::scope_index::main::identity_failure_message::identity_failure_message; use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; -use sqlbuild_scopes::scope_report::main::browse_scope_folders::browse_scope_folders; -use sqlbuild_scopes::scope_report::main::build_scope_report::build_scope_report; use sqlbuild_scopes::scope_report::main::cached_completeness::cached_completeness; use sqlbuild_scopes::scope_report::main::cached_declaration_values::cached_declaration_values; use sqlbuild_scopes::scope_report::main::cached_projection_index::cached_projection_index; -use sqlbuild_scopes::scope_report::main::explain_scope_declaration::explain_scope_declaration; -use sqlbuild_scopes::scope_report::main::list_scope_declarations::list_scope_declarations; -use sqlbuild_scopes::scope_report::main::preview_scope_move::preview_scope_move; -use sqlbuild_scopes::scope_report::main::rebound_scope_usages::rebound_scope_usages; use sqlbuild_scopes::scope_report::main::scope_metadata_projection::scope_metadata_projection; use sqlbuild_scopes::scope_report::main::serialize_scope_result::serialize_scope_result; use sqlbuild_scopes::scope_report::models::{ - DeclarationValues, DiagnosticFacts, ExplainRequest, FolderRequest, MoveRequest, - ProjectionIndex, ReportDiagnostic, ReportFailure, ReportRequest, ReportSource, ReportValues, - ScopeResult, + DeclarationValues, DiagnosticFacts, FolderRequest, ProjectionIndex, ReportDiagnostic, + ReportFailure, ReportSource, ReportValues, ScopeResult, }; use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; -use crate::scopes::_helpers::index::scope_objects::read_completeness; -use crate::scopes::_helpers::index::scope_objects::read_declaration; -use crate::scopes::_helpers::index::scope_objects::read_index; -use crate::scopes::_helpers::reports::scope_report_objects::browse_object; -use crate::scopes::_helpers::reports::scope_report_objects::diagnostics_object; -use crate::scopes::_helpers::reports::scope_report_objects::explanation_object; -use crate::scopes::_helpers::reports::scope_report_objects::json_to_plain; -use crate::scopes::_helpers::reports::scope_report_objects::json_to_python; -use crate::scopes::_helpers::reports::scope_report_objects::list_object; -use crate::scopes::_helpers::reports::scope_report_objects::move_object; -use crate::scopes::_helpers::reports::scope_report_objects::read_declaration_target; -use crate::scopes::_helpers::reports::scope_report_objects::read_filters; -use crate::scopes::_helpers::reports::scope_report_objects::read_optional_target; -use crate::scopes::_helpers::reports::scope_report_objects::read_result; -use crate::scopes::_helpers::reports::scope_report_objects::read_target; -use crate::scopes::_helpers::reports::scope_report_objects::report_object; +use crate::scopes::_helpers::index::scope_objects::{ + read_completeness, read_declaration, read_index, +}; +use crate::scopes::_helpers::reports::scope_report_objects::{read_filters, read_optional_target}; use crate::scopes::models::ScopeClasses; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_base::records::types::RecordId; pub(crate) type PyObject = Py; const CURRENT_FOLDER: &str = "."; /// One lookup's native lookup and every declaration's values in input order. -struct LookupReports { - lookup: Py, +pub(crate) struct LookupReports { + pub(crate) lookup: Py, values: ReportValues, } impl LookupReports { - fn source<'a>(&'a self, lookup: &'a NativeScopeLookup) -> ReportSource<'a> { + pub(crate) fn source<'a>(&'a self, lookup: &'a NativeScopeLookup) -> ReportSource<'a> { ReportSource { records: &lookup.records, lookup: &lookup.lookup, @@ -74,7 +58,7 @@ impl LookupReports { } /// The reports of a `ScopeLookup`: its native lookup with its canonical declarations' values. -fn lookup_reports(lookup: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn lookup_reports(lookup: &Bound<'_, PyAny>) -> PyResult { let index: Bound<'_, PyAny> = native_field(lookup, "index")?; let native: Py = native_field(lookup, "native")?.extract()?; let rows: Vec = native_field(&index, "declarations")? @@ -100,14 +84,14 @@ fn lookup_reports(lookup: &Bound<'_, PyAny>) -> PyResult { }) } -fn scope_error(py: Python<'_>, class: ErrorId, message: String) -> PyResult { +pub(crate) fn scope_error(py: Python<'_>, class: ErrorId, message: String) -> PyResult { Ok(PyErr::from_value( error_class(py, class)?.call1((message,))?, )) } /// The error Python raises for a failed query. -fn failure_error(py: Python<'_>, failure: &ReportFailure) -> PyResult { +pub(crate) fn failure_error(py: Python<'_>, failure: &ReportFailure) -> PyResult { match failure { ReportFailure::Path(rejected) => scope_error( py, @@ -124,90 +108,21 @@ fn failure_error(py: Python<'_>, failure: &ReportFailure) -> PyResult { } } -fn path_text(value: Option>) -> PyResult> { +pub(crate) fn path_text(value: Option>) -> PyResult> { value .filter(|value| !value.is_none()) .map(|value| value.str()?.extract()) .transpose() } -fn raised(diagnostics: Vec) -> Vec { +pub(crate) fn raised(diagnostics: Vec) -> Vec { diagnostics .into_iter() .map(ReportDiagnostic::Raised) .collect() } -/// `query_scope_report`: one existing target or prospective path; invalid paths raise. -pub(crate) fn scope_report( - py: Python<'_>, - lookup: &Bound<'_, PyAny>, - request: ReportArguments<'_>, -) -> PyResult { - let ReportArguments { - target, - at, - directory, - filters, - } = request; - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - let selected: Bound<'_, PyAny> = match filters.filter(|filters| !filters.is_none()) { - Some(filters) => filters, - None => classes.model(RecordId::ScopeReportFilters)?.call0()?, - }; - let reports: LookupReports = lookup_reports(lookup)?; - let request: ReportRequest = ReportRequest { - target: read_optional_target(&classes, target.as_ref())?, - at: path_text(at)?, - directory, - filters: read_filters(&selected)?, - }; - let native: &NativeScopeLookup = reports.lookup.get(); - match build_scope_report(reports.source(native), &request) { - Ok(report) => { - let diagnostics: Bound<'_, PyAny> = - native_field(&native_field(lookup, "index")?, "diagnostics")?; - Ok(report_object(&classes, report, (&selected, &diagnostics))?.unbind()) - } - Err(failure) => Err(failure_error(py, &failure)?), - } -} - -/// `explain_scope_declaration`: one qualified declaration in an optional context. -pub(crate) fn scope_explain( - py: Python<'_>, - lookup: &Bound<'_, PyAny>, - request: ExplainArguments<'_>, -) -> PyResult<(PyObject, PyObject)> { - let ExplainArguments { - declaration, - target, - at, - directory, - } = request; - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - let reports: LookupReports = lookup_reports(lookup)?; - let request: ExplainRequest = ExplainRequest { - declaration: read_declaration_target(&classes, &declaration)?, - target: read_optional_target(&classes, target.as_ref())?, - at: path_text(at)?, - directory, - }; - let native: &NativeScopeLookup = reports.lookup.get(); - match explain_scope_declaration(reports.source(native), &request) { - Ok((explanation, diagnostics)) => { - let index_diagnostics: Bound<'_, PyAny> = - native_field(&native_field(lookup, "index")?, "diagnostics")?; - Ok(( - explanation_object(&classes, explanation)?.unbind(), - diagnostics_object(&classes, &index_diagnostics, raised(diagnostics))?.unbind(), - )) - } - Err(failure) => Err(failure_error(py, &failure)?), - } -} - -fn folder_request( +pub(crate) fn folder_request( classes: &ScopeClasses<'_>, location: (String, Option>, bool), filters: Option>, @@ -224,95 +139,10 @@ fn folder_request( }) } -/// `browse_scope_folders`: the direct child declaration-definition folders of `folder`. -pub(crate) fn scope_browse( - py: Python<'_>, - lookup: &Bound<'_, PyAny>, - request: (String, Option>, bool), -) -> PyResult { - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - let reports: LookupReports = lookup_reports(lookup)?; - let request: FolderRequest = folder_request(&classes, request, None)?; - let native: &NativeScopeLookup = reports.lookup.get(); - let result = browse_scope_folders(reports.source(native), &request); - let diagnostics: Bound<'_, PyAny> = - native_field(&native_field(lookup, "index")?, "diagnostics")?; - Ok(browse_object(&classes, result, &diagnostics)?.unbind()) -} - -/// `list_scope_declarations`: declarations recursively beneath one definition folder. -pub(crate) fn scope_list( - py: Python<'_>, - lookup: &Bound<'_, PyAny>, - request: ListArguments<'_>, -) -> PyResult { - let ListArguments { - folder, - filters, - target, - target_is_prospective, - } = request; - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - let reports: LookupReports = lookup_reports(lookup)?; - let request: FolderRequest = - folder_request(&classes, (folder, target, target_is_prospective), filters)?; - let native: &NativeScopeLookup = reports.lookup.get(); - match list_scope_declarations(reports.source(native), &request) { - Ok(result) => { - let diagnostics: Bound<'_, PyAny> = - native_field(&native_field(lookup, "index")?, "diagnostics")?; - Ok(list_object(&classes, result, &diagnostics)?.unbind()) - } - Err(failure) => Err(failure_error(py, &failure)?), - } -} - -/// `preview_scope_move`: visibility deltas of a resource move, without mutating the lookup. -#[pyfunction] -#[pyo3(name = "preview_scope_move", signature = (*, lookup, resource, destination))] -pub(crate) fn scope_move_preview( - py: Python<'_>, - lookup: &Bound<'_, PyAny>, - resource: &Bound<'_, PyAny>, - destination: &Bound<'_, PyAny>, -) -> PyResult<(Option, PyObject)> { - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - let reports: LookupReports = lookup_reports(lookup)?; - let request: MoveRequest = MoveRequest { - resource: read_target(&classes, resource)?, - destination: destination.str()?.extract()?, - }; - let native: &NativeScopeLookup = reports.lookup.get(); - match preview_scope_move(reports.source(native), &request) { - Ok((preview, diagnostics)) => { - let index_diagnostics: Bound<'_, PyAny> = - native_field(&native_field(lookup, "index")?, "diagnostics")?; - Ok(( - preview - .map(|preview| Ok::<_, PyErr>(move_object(&classes, preview)?.unbind())) - .transpose()?, - diagnostics_object(&classes, &index_diagnostics, raised(diagnostics))?.unbind(), - )) - } - Err(failure) => Err(failure_error(py, &failure)?), - } -} - -/// Scoped declarations the resources at `path` use that resolve differently from `destination`. -pub(crate) fn scope_rebound_usages( - py: Python<'_>, - lookup: &Bound<'_, PyAny>, - paths: (&str, &str), -) -> PyResult>> { - let reports: LookupReports = lookup_reports(lookup)?; - let native: &NativeScopeLookup = reports.lookup.get(); - match rebound_scope_usages(reports.source(native), paths) { - Ok(rebound) => Ok(rebound), - Err(failure) => Err(failure_error(py, &failure)?), - } -} - -fn serialized(classes: &ScopeClasses<'_>, result: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn serialized( + classes: &ScopeClasses<'_>, + result: &Bound<'_, PyAny>, +) -> PyResult { let result: ScopeResult = read_result(classes, result)?; serialize_scope_result(&result).map_err(|failure| match failure { ReportFailure::Serialization(message) => PyValueError::new_err(message), @@ -320,25 +150,6 @@ fn serialized(classes: &ScopeClasses<'_>, result: &Bound<'_, PyAny>) -> PyResult }) } -/// `serialize_scope_report`: deterministic schema-version-two ASCII JSON of one result. -#[pyfunction] -#[pyo3(signature = (*, report))] -pub(crate) fn serialize_scope_report( - py: Python<'_>, - report: &Bound<'_, PyAny>, -) -> PyResult { - let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; - match serialized(&classes, report) { - Err(error) if error.is_instance_of::(py) => { - let message: String = error.value(py).str()?.extract()?; - let raised: PyErr = scope_error(py, ErrorId::Scope, message)?; - raised.set_cause(py, Some(error)); - Err(raised) - } - other => other, - } -} - /// `scope_metadata_projection` of one `ScopeIndex`, as the dictionary Python builds. #[pyfunction] #[pyo3(name = "scope_metadata_projection", signature = (*, index))] @@ -346,12 +157,7 @@ pub(crate) fn metadata_projection(py: Python<'_>, index: &Bound<'_, PyAny>) -> P json_to_python(py, projection_json(index)?, false) } -/// `plain_value` of `scope_metadata_projection(index=index)`, without a Python round trip. -pub(crate) fn metadata_projection_value(index: &Bound<'_, PyAny>) -> PyResult { - json_to_plain(projection_json(index)?) -} - -fn projection_json(index: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn projection_json(index: &Bound<'_, PyAny>) -> PyResult { let index: ProjectionIndex = cached_projection_index(read_index(index)?); Ok(scope_metadata_projection(&index)) } @@ -366,7 +172,7 @@ fn optional<'py>( /// Python's `query_scope_report`. #[pyfunction] #[pyo3(name = "query_scope_report", signature = (**options))] -fn query_scope_report_binding<'py>( +pub(crate) fn query_scope_report_binding<'py>( py: Python<'py>, options: Option>, ) -> PyResult { @@ -391,7 +197,7 @@ fn query_scope_report_binding<'py>( /// Python's `explain_scope_declaration`. #[pyfunction] #[pyo3(name = "explain_scope_declaration", signature = (**options))] -fn explain_scope_declaration_binding<'py>( +pub(crate) fn explain_scope_declaration_binding<'py>( py: Python<'py>, options: Option>, ) -> PyResult<(PyObject, PyObject)> { @@ -416,7 +222,7 @@ fn explain_scope_declaration_binding<'py>( /// Python's `browse_scope_folders`. #[pyfunction] #[pyo3(name = "browse_scope_folders", signature = (**options))] -fn browse_scope_folders_binding<'py>( +pub(crate) fn browse_scope_folders_binding<'py>( py: Python<'py>, options: Option>, ) -> PyResult { @@ -444,7 +250,7 @@ fn browse_scope_folders_binding<'py>( /// Python's `list_scope_declarations`. #[pyfunction] #[pyo3(name = "list_scope_declarations", signature = (**options))] -fn list_scope_declarations_binding<'py>( +pub(crate) fn list_scope_declarations_binding<'py>( py: Python<'py>, options: Option>, ) -> PyResult { @@ -468,14 +274,3 @@ fn list_scope_declarations_binding<'py>( }, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(query_scope_report_binding, module)?)?; - module.add_function(wrap_pyfunction!(explain_scope_declaration_binding, module)?)?; - module.add_function(wrap_pyfunction!(browse_scope_folders_binding, module)?)?; - module.add_function(wrap_pyfunction!(list_scope_declarations_binding, module)?)?; - module.add_function(wrap_pyfunction!(scope_move_preview, module)?)?; - module.add_function(wrap_pyfunction!(serialize_scope_report, module)?)?; - module.add_function(wrap_pyfunction!(metadata_projection, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs index 59bb9c9aa3..194f2fe490 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs @@ -1,12 +1,15 @@ //! Register the scope validation functions and classes. -use crate::scopes::_helpers::checks::scope_validation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::checks::scope_validation::scope_placement_validated_index; +use crate::scopes::_helpers::checks::scope_validation::scope_relocated_declarations; +use crate::scopes::main::checks::validate_scope_index::validate_scope_index; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scope validation functions and classes. -#[inline] pub fn register_scope_validation(module: &Bound<'_, PyModule>) -> PyResult<()> { - scope_validation::register(module) + module.add_function(wrap_pyfunction!(validate_scope_index, module)?)?; + module.add_function(wrap_pyfunction!(scope_placement_validated_index, module)?)?; + module.add_function(wrap_pyfunction!(scope_relocated_declarations, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs index 03b3da25b7..441a85470d 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs @@ -1,12 +1,20 @@ //! Register the scope visibility functions and classes. -use crate::scopes::_helpers::checks::scope_visibility; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::checks::scope_visibility::resolve_scope_declaration_visibility; +use crate::scopes::_helpers::checks::scope_visibility::resolve_scope_visibility; +use crate::scopes::_helpers::checks::scope_visibility::scope_query_target; +use crate::scopes::_helpers::checks::scope_visibility::scope_resolve_path_visibility; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scope visibility functions and classes. -#[inline] pub fn register_scope_visibility(module: &Bound<'_, PyModule>) -> PyResult<()> { - scope_visibility::register(module) + module.add_function(wrap_pyfunction!(scope_query_target, module)?)?; + module.add_function(wrap_pyfunction!(resolve_scope_visibility, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_scope_declaration_visibility, + module + )?)?; + module.add_function(wrap_pyfunction!(scope_resolve_path_visibility, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/relocated_declarations.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/relocated_declarations.rs index 1f67ce386f..1d60cd9dba 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/checks/relocated_declarations.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/relocated_declarations.rs @@ -1,17 +1,60 @@ //! Relocated records after every move in `moves`, and per record whether it lacks a needed group. -use crate::scopes::_helpers::checks::scope_validation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyTuple; +use crate::scopes::_helpers::checks::scope_validation::RUNTIME_USAGE; +use crate::scopes::_helpers::checks::scope_validation::replace; +use crate::scopes::_helpers::index::scope_objects::{ + read_completeness, read_placement_index, root_object, tuple_attribute, +}; +use crate::scopes::models::ScopeClasses; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::{PyTuple, PyTupleMethods}; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_scopes::placement::main::relocate_for_resource_moves::relocate_for_resource_moves; +use sqlbuild_scopes::placement::models::{PlacementIndex, Relocation}; use sqlbuild_scopes::scope_index::models::ResourceIdentity; /// Relocated records after every move in `moves`, and per record whether it lacks a needed group. -#[inline] pub fn relocated_declarations<'py>( index: &Bound<'py, PyAny>, moves: &[(ResourceIdentity, String)], ) -> PyResult, Vec)>> { - scope_validation::relocated_declarations(index, moves) + let py: Python<'py> = index.py(); + if !read_completeness(index)?[RUNTIME_USAGE] { + return Ok(None); + } + let placement: PlacementIndex = read_placement_index(index)?; + let relocations: Option> = py + .compiler_detach(|| Ok(relocate_for_resource_moves(&placement, moves))) + .map_err(compiler_error)?; + let Some(relocations) = relocations else { + return Ok(None); + }; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let records: Bound<'_, PyTuple> = tuple_attribute(index, "declarations")?; + let mut moved: Vec> = Vec::with_capacity(relocations.len()); + let mut grouping: Vec = Vec::with_capacity(relocations.len()); + for item in relocations { + let owning_path: Bound<'_, PyAny> = item.owning_path.into_bound_py_any(py)?; + let record: Bound<'py, PyAny> = replace( + &records.get_item(item.declaration)?, + &[ + ("path", item.path.into_bound_py_any(py)?), + ( + "scope", + classes.member(RecordId::ScopeKind, item.scope.as_str())?, + ), + ( + "ownership_root", + root_object(&classes, &item.ownership_root)?, + ), + ("owning_path", owning_path), + ], + )?; + moved.push(record); + grouping.push(item.requires_group); + } + Ok(Some((PyTuple::new(py, moved)?, grouping))) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/scope_index_with_compile_usages.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/scope_index_with_compile_usages.rs index 8eb10e7943..1ed107998c 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/checks/scope_index_with_compile_usages.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/scope_index_with_compile_usages.rs @@ -1,14 +1,62 @@ //! Compile inputs' canonical index with all usage and placement facts. -use crate::scopes::_helpers::checks::scope_validation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::checks::scope_validation::COMPILE_USAGE_GROUPS; +use crate::scopes::_helpers::checks::scope_validation::scope_placement_validated_index; +use crate::scopes::_helpers::checks::scope_validation::usage_records; +use crate::scopes::_helpers::relationships::compile_usages::model_schema_scope_usages; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Compile inputs' canonical index with all usage and placement facts. -#[inline] pub fn scope_index_with_compile_usages<'py>( inputs: &Bound<'py, PyAny>, ) -> PyResult> { - scope_validation::scope_index_with_compile_usages(inputs) + let py: Python<'py> = inputs.py(); + let scope: Bound<'py, PyAny> = native_field(inputs, "declaration_scope")?; + let resolver: Bound<'py, PyAny> = if scope.is_none() { + scope + } else { + native_field(&scope, "resolver")? + }; + let schema_files: Bound<'py, PyAny> = native_field( + &native_field(inputs, "discovered_inputs")?, + "model_schema_files", + )?; + let mut usages: Vec> = Vec::new(); + usages.extend(usage_records( + model_schema_scope_usages(&schema_files, &resolver)?.as_any(), + )?); + let digests: bool = sqlbuild_python_base::compile_context::main::delta::capturing::capturing(); + for model in native_field(inputs, "model_inputs")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let macro_usages: Vec> = + usage_records(&native_field(&model, "macro_usages")?)?; + let declaration_usages: Vec> = + usage_records(&native_field(&model, "declaration_usages")?)?; + if digests { + sqlbuild_python_base::compile_context::main::image_facts::record_model_usages::record_model_usages( + &model, + (¯o_usages, &declaration_usages), + )?; + } + usages.extend(macro_usages); + usages.extend(declaration_usages); + } + for group in COMPILE_USAGE_GROUPS { + for item in native_field(inputs, group)?.try_iter()? { + usages.extend(usage_records(&native_field(&item?, "declaration_usages")?)?); + } + } + let enforce_placement: bool = native_field( + &native_field(&native_field(inputs, "project_config")?, "scopes")?, + "enforce_placement", + )? + .extract()?; + Ok(scope_placement_validated_index( + &native_field(inputs, "scope_index")?, + enforce_placement, + Some(PyTuple::new(py, usages)?), + )? + .into_bound(py)) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/scope_tolerant_index.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/scope_tolerant_index.rs index 82e2186ba0..896d8d70de 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/checks/scope_tolerant_index.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/scope_tolerant_index.rs @@ -1,11 +1,7 @@ //! The tolerant index: fault diagnostics, prospective resources and partial completeness. use crate::scopes::_helpers::checks::scope_tolerant; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; /// The tolerant index: fault diagnostics, prospective resources and partial completeness. #[inline] diff --git a/crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs b/crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs index 117154f15d..60af7d0c08 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs @@ -1,12 +1,39 @@ //! Python's `validate_scope_index`: raise one `ScopeValidationError` for its error diagnostics. -use crate::scopes::_helpers::checks::scope_validation; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyErr; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_base::records::types::RecordId; /// Python's `validate_scope_index`: raise one `ScopeValidationError` for its error diagnostics. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, index))] pub fn validate_scope_index(index: &Bound<'_, PyAny>) -> Result<(), PyErr> { - scope_validation::validate_scope_index(index) + let py: Python<'_> = index.py(); + let error: Bound<'_, PyAny> = enum_member(py, RecordId::ScopeDiagnosticSeverity, "ERROR")?; + let mut errors: Vec> = Vec::new(); + for item in native_field(index, "diagnostics")?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + if native_field(&item, "severity")?.is(&error) { + errors.push(item); + } + } + if errors.is_empty() { + return Ok(()); + } + let mut message: String = String::from("Invalid declaration scope index:"); + for item in &errors { + let code: String = native_field(&native_field(item, "code")?, "value")?.extract()?; + let text: String = native_field(item, "message")?.extract()?; + message.push_str(&format!("\n[{code}] {text}")); + } + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("diagnostics", PyTuple::new(py, errors)?)?; + Err(PyErr::from_value( + error_class(py, ErrorId::ScopeValidation)?.call((message,), Some(&options))?, + )) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/declarations/register_declaration_contexts.rs b/crates/sqlbuild-python-front/src/scopes/main/declarations/register_declaration_contexts.rs index b613d65b38..7aad982058 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/declarations/register_declaration_contexts.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/declarations/register_declaration_contexts.rs @@ -1,12 +1,10 @@ //! Register the declaration contexts functions and classes. -use crate::scopes::_helpers::declarations::declaration_contexts; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::models::NativeDeclarationContexts; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the declaration contexts functions and classes. -#[inline] pub fn register_declaration_contexts(module: &Bound<'_, PyModule>) -> PyResult<()> { - declaration_contexts::register(module) + module.add_class::()?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/build_scope_lookup.rs b/crates/sqlbuild-python-front/src/scopes/main/index/build_scope_lookup.rs index d9260694d9..4233aa2229 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/build_scope_lookup.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/build_scope_lookup.rs @@ -1,14 +1,104 @@ //! Python's `build_scope_lookup`: the canonical index, its mappings and the native lookup. -use crate::scopes::_helpers::index::scope_lookup; -use pyo3::prelude::Bound; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::index::scope_lookup::CanonicalPositions; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::index::scope_lookup::grouped; +use crate::scopes::_helpers::index::scope_lookup::inverse; +use crate::scopes::_helpers::index::scope_lookup::path_grouped; +use crate::scopes::_helpers::index::scope_lookup::picked; +use crate::scopes::_helpers::index::scope_lookup::scope_path_error; +use crate::scopes::_helpers::index::scope_lookup::visibility_index; +use crate::scopes::_helpers::index::scope_objects::{read_index, tuple_attribute}; +use crate::scopes::models::ScopeClasses; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyTuple; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_scopes::scope_index::main::rejected_path_message::rejected_path_message; +use sqlbuild_scopes::scope_index::main::report_lookup::report_lookup; +use sqlbuild_scopes::scope_index::models::{ReportLookup, ReportRecords}; +use sqlbuild_scopes::scope_report::main::cached_report_records::cached_report_records; /// Python's `build_scope_lookup`: the canonical index, its mappings and the native lookup. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, index))] pub fn build_scope_lookup(py: Python<'_>, index: &Bound<'_, PyAny>) -> PyResult> { - scope_lookup::build_scope_lookup(py, index) + let records: ReportRecords = cached_report_records(&read_index(index)?); + let built = py + .compiler_detach(|| Ok(report_lookup(&records))) + .map_err(compiler_error)?; + let lookup: ReportLookup = match built { + Ok(lookup) => lookup, + Err(rejected) => return Err(scope_path_error(py, rejected_path_message(&rejected))?), + }; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let resources: Bound<'_, PyTuple> = tuple_attribute(index, "resources")?; + let declarations: Bound<'_, PyTuple> = tuple_attribute(index, "declarations")?; + let usages: Bound<'_, PyTuple> = tuple_attribute(index, "usages")?; + let grants: Bound<'_, PyTuple> = tuple_attribute(index, "grants")?; + let canonical_declarations: Bound<'_, PyTuple> = picked(&declarations, &lookup.declarations)?; + let canonical_index: Bound<'_, PyAny> = classes.model(RecordId::ScopeIndex)?.call1(( + picked(&tuple_attribute(index, "ownership_roots")?, &lookup.roots)?, + picked(&resources, &lookup.resources)?, + &canonical_declarations, + picked(&usages, &lookup.usages)?, + picked(&grants, &lookup.grants)?, + native_field(index, "visibility")?, + native_field(index, "inaccessible")?, + picked(&tuple_attribute(index, "diagnostics")?, &lookup.diagnostics)?, + native_field(index, "completeness")?, + ))?; + let groups = &lookup.groups; + let mappings: [Bound<'_, PyAny>; 6] = [ + grouped(&groups.resources, &resources, "identity")?, + path_grouped( + &groups.resources_by_path, + &resources, + &lookup.resource_paths, + )?, + grouped(&groups.declarations, &declarations, "identity")?, + grouped(&groups.usages_by_consumer, &usages, "consumer")?, + grouped(&groups.usages_by_declaration, &usages, "declaration")?, + grouped(&groups.grants_by_resource, &grants, "resource")?, + ]; + let visibility: Bound<'_, PyAny> = + visibility_index(&classes, &canonical_declarations, &lookup.visibility)?; + let canonical: CanonicalPositions = CanonicalPositions { + resources: inverse(&lookup.resources), + declarations: inverse(&lookup.declarations), + grants: inverse(&lookup.grants), + }; + let native: Py = Py::new( + py, + NativeScopeLookup { + records, + lookup, + canonical, + }, + )?; + let [ + by_identity, + by_path, + by_declaration, + by_consumer, + by_used, + by_resource, + ] = mappings; + Ok(classes + .model(RecordId::ScopeLookup)? + .call1(( + canonical_index, + by_identity, + by_path, + by_declaration, + by_consumer, + by_used, + by_resource, + visibility, + native, + ))? + .unbind()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/open_native_scope_index.rs b/crates/sqlbuild-python-front/src/scopes/main/index/open_native_scope_index.rs index 2196717847..c26ad856bc 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/open_native_scope_index.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/open_native_scope_index.rs @@ -1,19 +1,46 @@ //! Python's `open_native_scope_index`: the natively built index of discovered inputs. -use crate::scopes::_helpers::index::scope_index; +use crate::scopes::_helpers::index::scope_index::Walk; +use crate::scopes::_helpers::index::scope_index::materialise; +use crate::scopes::_helpers::index::scope_index::unindexable; use crate::scopes::models::NativeScopeIndex; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyTuple; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_scopes::scope_index::main::build_scope_index::build_scope_index; +use sqlbuild_scopes::scope_index::models::{ScopeDeferral, ScopeIndex, ScopeInputs}; /// Python's `open_native_scope_index`: the natively built index of discovered inputs. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, loaded_macros, static_macros = None))] pub fn open_native_scope_index( py: Python<'_>, discovered_inputs: &Bound<'_, PyAny>, loaded_macros: &Bound<'_, PyAny>, static_macros: Option>, ) -> PyResult { - scope_index::open_native_scope_index(py, discovered_inputs, loaded_macros, static_macros) + let static_macros: Bound<'_, PyAny> = match static_macros { + Some(static_macros) => static_macros, + None => PyTuple::empty(py).into_any(), + }; + let static_macros: &Bound<'_, PyAny> = &static_macros; + let mut walk: Walk<'_> = Walk::new(py)?; + walk.add_models(discovered_inputs)?; + walk.add_resources(discovered_inputs)?; + walk.add_declaration_files(discovered_inputs)?; + walk.add_named_declarations(discovered_inputs)?; + walk.add_macros(discovered_inputs, (loaded_macros, static_macros))?; + if !walk.indexable { + return Err(unindexable(py)?); + } + let inputs: ScopeInputs = std::mem::take(&mut walk.inputs); + let built: Result = py + .compiler_detach(|| Ok(build_scope_index(inputs))) + .map_err(compiler_error)?; + let Ok(native) = built else { + return Err(unindexable(py)?); + }; + materialise(walk, native) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/project_fingerprint.rs b/crates/sqlbuild-python-front/src/scopes/main/index/project_fingerprint.rs index ac5ddfb553..0a379823e8 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/project_fingerprint.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/project_fingerprint.rs @@ -1,12 +1,26 @@ //! The scope index fingerprint of `project_dir`, `None` when it cannot be read. -use crate::scopes::_helpers::index::scope_cache; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::index::scope_cache::sqlbuild_version; +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_scopes::scope_cache::main::project_fingerprint_inputs::project_fingerprint_inputs; +use sqlbuild_scopes::scope_cache::main::scope_fingerprint::scope_fingerprint; +use sqlbuild_scopes::scope_cache::models::{Fingerprint, FingerprintInputs}; use std::path::PathBuf; /// The scope index fingerprint of `project_dir`, `None` when it cannot be read. -#[inline] pub fn project_fingerprint(py: Python<'_>, project_dir: PathBuf) -> PyResult> { - scope_cache::project_fingerprint(py, project_dir) + let inputs: FingerprintInputs = project_fingerprint_inputs(sqlbuild_version(py)?); + let fingerprint: Fingerprint = py + .compiler_detach(|| Ok(scope_fingerprint(&project_dir, &inputs))) + .map_err(compiler_error)?; + match fingerprint { + Fingerprint::Hash(hash) => Ok(Some(hash)), + Fingerprint::Unavailable => Ok(None), + Fingerprint::Unserializable => Err(PyTypeError::new_err( + "a scope-relevant config value is not JSON serializable", + )), + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/read_scope_index_cache.rs b/crates/sqlbuild-python-front/src/scopes/main/index/read_scope_index_cache.rs index 146846e797..e481760b23 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/read_scope_index_cache.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/read_scope_index_cache.rs @@ -1,18 +1,29 @@ //! The verified cached `ScopeIndex` at `path` for `fingerprint`; every fault is a miss. -use crate::scopes::_helpers::index::scope_cache; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::index::scope_objects::index_object; +use crate::scopes::models::ScopeClasses; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Py, PyAny, PyResult, Python}; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_scopes::scope_cache::main::read_scope_cache::read_scope_cache; use std::path::PathBuf; /// The verified cached `ScopeIndex` at `path` for `fingerprint`; every fault is a miss. -#[inline] pub fn read_scope_index_cache( py: Python<'_>, path: PathBuf, fingerprint: String, ) -> PyResult>> { - scope_cache::read_scope_index_cache(py, path, fingerprint) + let cached = py + .compiler_detach(|| Ok(read_scope_cache(&path, &fingerprint))) + .map_err(compiler_error)?; + let Ok(index) = cached else { + return Ok(None); + }; + match index_object(&ScopeClasses::new(py)?, &index) { + Ok(object) => Ok(Some(object.unbind())), + Err(error) if error.is_instance_of::(py) => Ok(None), + Err(error) => Err(error), + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_cache.rs b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_cache.rs index 4dcdf09e70..aa70b4cb80 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_cache.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_cache.rs @@ -1,12 +1,11 @@ //! Register the scope cache functions and classes. -use crate::scopes::_helpers::index::scope_cache; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::index::scope_cache::scope_index_fingerprint; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scope cache functions and classes. -#[inline] pub fn register_scope_cache(module: &Bound<'_, PyModule>) -> PyResult<()> { - scope_cache::register(module) + module.add_function(wrap_pyfunction!(scope_index_fingerprint, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_index.rs b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_index.rs index 1194dee508..3e901df83f 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_index.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_index.rs @@ -1,12 +1,15 @@ //! Register the scope index functions and classes. -use crate::scopes::_helpers::index::scope_index; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::index::scope_index::build_scope_index_binding; +use crate::scopes::main::index::open_native_scope_index::open_native_scope_index; +use crate::scopes::models::NativeScopeIndex; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scope index functions and classes. -#[inline] pub fn register_scope_index(module: &Bound<'_, PyModule>) -> PyResult<()> { - scope_index::register(module) + module.add_class::()?; + module.add_function(wrap_pyfunction!(open_native_scope_index, module)?)?; + module.add_function(wrap_pyfunction!(build_scope_index_binding, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_lookup.rs b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_lookup.rs index 4c3e76c30d..11b77031f3 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_lookup.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/register_scope_lookup.rs @@ -1,12 +1,15 @@ //! Register the scope lookup functions and classes. -use crate::scopes::_helpers::index::scope_lookup; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::index::scope_lookup::scope_normalize_path; +use crate::scopes::main::index::build_scope_lookup::build_scope_lookup; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scope lookup functions and classes. -#[inline] pub fn register_scope_lookup(module: &Bound<'_, PyModule>) -> PyResult<()> { - scope_lookup::register(module) + module.add_class::()?; + module.add_function(wrap_pyfunction!(build_scope_lookup, module)?)?; + module.add_function(wrap_pyfunction!(scope_normalize_path, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/index/write_scope_index_cache.rs b/crates/sqlbuild-python-front/src/scopes/main/index/write_scope_index_cache.rs index 1f3e8858d2..64c5a6faf7 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/index/write_scope_index_cache.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/index/write_scope_index_cache.rs @@ -1,19 +1,31 @@ //! Atomically write one complete `ScopeIndex` for `fingerprint`; whether the write landed. -use crate::scopes::_helpers::index::scope_cache; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::index::scope_objects::{read_completeness, read_index}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_scopes::scope_cache::main::encode_scope_cache::encode_scope_cache; +use sqlbuild_scopes::scope_cache::main::write_scope_cache::write_scope_cache; +use sqlbuild_scopes::scope_cache::models::CachedIndex; use std::path::PathBuf; /// Atomically write one complete `ScopeIndex` for `fingerprint`; whether the write landed. -#[inline] pub fn write_scope_index_cache( py: Python<'_>, location: (PathBuf, String), fingerprint: String, index: &Bound<'_, PyAny>, ) -> PyResult { - scope_cache::write_scope_index_cache(py, location, fingerprint, index) + if !read_completeness(index)?.iter().all(|done| *done) { + return Ok(false); + } + let index: CachedIndex = read_index(index)?; + let (directory, file_name) = location; + py.compiler_detach(|| { + Ok(match encode_scope_cache(&index, &fingerprint) { + Ok(envelope) => matches!(write_scope_cache(&directory, &file_name, &envelope), Ok(())), + Err(_) => false, + }) + }) + .map_err(compiler_error) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/relationships/register_compile_usages.rs b/crates/sqlbuild-python-front/src/scopes/main/relationships/register_compile_usages.rs index 2b36f57b36..782f59408b 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/relationships/register_compile_usages.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/relationships/register_compile_usages.rs @@ -1,12 +1,11 @@ //! Register the compile usages functions and classes. -use crate::scopes::_helpers::relationships::compile_usages; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::relationships::compile_usages::model_schema_scope_usages; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the compile usages functions and classes. -#[inline] pub fn register_compile_usages(module: &Bound<'_, PyModule>) -> PyResult<()> { - compile_usages::register(module) + module.add_function(wrap_pyfunction!(model_schema_scope_usages, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/relationships/register_relationship_names.rs b/crates/sqlbuild-python-front/src/scopes/main/relationships/register_relationship_names.rs index 03781fc7b2..7461a22953 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/relationships/register_relationship_names.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/relationships/register_relationship_names.rs @@ -1,12 +1,13 @@ //! Register the relationship names functions and classes. -use crate::scopes::_helpers::relationships::relationship_names; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::relationships::relationship_names::scope_expected_model_names; +use crate::scopes::_helpers::relationships::relationship_names::scope_test_ctes; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the relationship names functions and classes. -#[inline] pub fn register_relationship_names(module: &Bound<'_, PyModule>) -> PyResult<()> { - relationship_names::register(module) + module.add_function(wrap_pyfunction!(scope_expected_model_names, module)?)?; + module.add_function(wrap_pyfunction!(scope_test_ctes, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/json_to_python.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/json_to_python.rs index ff32cbaeb1..cfbb87fe85 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/json_to_python.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/json_to_python.rs @@ -1,13 +1,44 @@ //! A JSON value as Python builds it; arrays become tuples or lists. -use crate::scopes::_helpers::reports::scope_report_objects; use crate::scopes::_helpers::reports::scope_report_objects::PyObject; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use sqlbuild_core::json::models::JsonValue; /// A JSON value as Python builds it; arrays become tuples or lists. -#[inline] pub fn json_to_python(py: Python<'_>, value: JsonValue, tuples: bool) -> PyResult { - scope_report_objects::json_to_python(py, value, tuples) + match value { + JsonValue::Null => Ok(py.None()), + JsonValue::Bool(flag) => flag.into_py_any(py), + JsonValue::Integer(integer) => match integer.to_i128() { + Some(number) => number.into_py_any(py), + None => Ok(py + .import("builtins")? + .getattr("int")? + .call1((integer.as_str(),))? + .unbind()), + }, + JsonValue::Float(number) => number.into_py_any(py), + JsonValue::String(text) => text.into_py_any(py), + JsonValue::Utf16(units) => String::from_utf16_lossy(&units).into_py_any(py), + JsonValue::Array(items) => { + let values: Vec = items + .into_iter() + .map(|item| json_to_python(py, item, tuples)) + .collect::>()?; + if tuples { + PyTuple::new(py, values)?.into_py_any(py) + } else { + PyList::new(py, values)?.into_py_any(py) + } + } + JsonValue::Object(entries) => { + let dictionary: Bound<'_, PyDict> = PyDict::new(py); + for (key, item) in entries { + dictionary.set_item(key, json_to_python(py, item, tuples)?)?; + } + dictionary.into_py_any(py) + } + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/metadata_projection_value.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/metadata_projection_value.rs index 9ccc5d8479..c789554831 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/metadata_projection_value.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/metadata_projection_value.rs @@ -1,12 +1,10 @@ //! `plain_value` of `scope_metadata_projection(index=index)`, without a Python round trip. -use crate::scopes::_helpers::reports::scope_reports; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::reports::scope_report_objects::json_to_plain; +use crate::scopes::_helpers::reports::scope_reports::projection_json; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// `plain_value` of `scope_metadata_projection(index=index)`, without a Python round trip. -#[inline] pub fn metadata_projection_value(index: &Bound<'_, PyAny>) -> PyResult { - scope_reports::metadata_projection_value(index) + json_to_plain(projection_json(index)?) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/read_result.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/read_result.rs index 255b493c51..aa1a4c8054 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/read_result.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/read_result.rs @@ -1,14 +1,37 @@ //! A `ScopeReport`, `ScopeBrowseResult` or, otherwise, `ScopeListResult` as its native result. -use crate::scopes::_helpers::reports::scope_report_objects; +use crate::scopes::_helpers::index::scope_objects::field; +use crate::scopes::_helpers::reports::scope_report_objects::read_declaration_reports; +use crate::scopes::_helpers::reports::scope_report_objects::read_diagnostics; +use crate::scopes::_helpers::reports::scope_report_objects::read_folder; +use crate::scopes::_helpers::reports::scope_report_objects::read_report; +use crate::scopes::_helpers::reports::scope_report_objects::read_section; use crate::scopes::models::ScopeClasses; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use sqlbuild_scopes::scope_report::models::ScopeResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_scopes::scope_report::models::{BrowseResult, ListResult, ScopeResult}; /// A `ScopeReport`, `ScopeBrowseResult` or, otherwise, `ScopeListResult` as its native result. -#[inline] pub fn read_result(classes: &ScopeClasses<'_>, result: &Bound<'_, PyAny>) -> PyResult { - scope_report_objects::read_result(classes, result) + if result.is_instance(&classes.model(RecordId::ScopeReport)?)? { + return Ok(ScopeResult::Report(Box::new(read_report(result)?))); + } + if result.is_instance(&classes.model(RecordId::ScopeBrowseResult)?)? { + return Ok(ScopeResult::Browse(BrowseResult { + folder: field(result, "folder")?, + folders: native_field(result, "folders")? + .try_iter()? + .map(|folder| read_folder(&folder?)) + .collect::>()?, + diagnostics: read_diagnostics(result)?, + complete: field(result, "complete")?, + })); + } + Ok(ScopeResult::List(ListResult { + folder: field(result, "folder")?, + declarations: read_declaration_reports(result, "declarations")?, + section: read_section(&native_field(result, "section")?)?, + diagnostics: read_diagnostics(result)?, + })) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs index fe8102070b..852a9c756a 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs @@ -1,12 +1,23 @@ //! Register the scope reports functions and classes. -use crate::scopes::_helpers::reports::scope_reports; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::scopes::_helpers::reports::scope_reports::browse_scope_folders_binding; +use crate::scopes::_helpers::reports::scope_reports::explain_scope_declaration_binding; +use crate::scopes::_helpers::reports::scope_reports::list_scope_declarations_binding; +use crate::scopes::_helpers::reports::scope_reports::metadata_projection; +use crate::scopes::_helpers::reports::scope_reports::query_scope_report_binding; +use crate::scopes::main::reports::scope_move_preview::scope_move_preview; +use crate::scopes::main::reports::serialize_scope_report::serialize_scope_report; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the scope reports functions and classes. -#[inline] pub fn register_scope_reports(module: &Bound<'_, PyModule>) -> PyResult<()> { - scope_reports::register(module) + module.add_function(wrap_pyfunction!(query_scope_report_binding, module)?)?; + module.add_function(wrap_pyfunction!(explain_scope_declaration_binding, module)?)?; + module.add_function(wrap_pyfunction!(browse_scope_folders_binding, module)?)?; + module.add_function(wrap_pyfunction!(list_scope_declarations_binding, module)?)?; + module.add_function(wrap_pyfunction!(scope_move_preview, module)?)?; + module.add_function(wrap_pyfunction!(serialize_scope_report, module)?)?; + module.add_function(wrap_pyfunction!(metadata_projection, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/report_scope_index_errors.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/report_scope_index_errors.rs index fb15112112..f67246a526 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/report_scope_index_errors.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/report_scope_index_errors.rs @@ -1,9 +1,7 @@ //! `report_scope_index_errors`: each error-severity finding of `index`, keyed for the collector. use crate::scopes::_helpers::reports::scope_errors; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// `report_scope_index_errors`: each error-severity finding of `index`, keyed for the collector. #[inline] diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_browse.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_browse.rs index 1a134fa5a9..05e2bfb47f 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_browse.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_browse.rs @@ -1,18 +1,29 @@ //! `browse_scope_folders`: the direct child declaration-definition folders of `folder`. -use crate::scopes::_helpers::reports::scope_reports; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::reports::scope_report_objects::browse_object; +use crate::scopes::_helpers::reports::scope_reports::LookupReports; use crate::scopes::_helpers::reports::scope_reports::PyObject; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::reports::scope_reports::folder_request; +use crate::scopes::_helpers::reports::scope_reports::lookup_reports; +use crate::scopes::models::ScopeClasses; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_scopes::scope_report::main::browse_scope_folders::browse_scope_folders; +use sqlbuild_scopes::scope_report::models::FolderRequest; /// `browse_scope_folders`: the direct child declaration-definition folders of `folder`. -#[inline] pub fn scope_browse( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: (String, Option>, bool), ) -> PyResult { - scope_reports::scope_browse(py, lookup, request) + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let reports: LookupReports = lookup_reports(lookup)?; + let request: FolderRequest = folder_request(&classes, request, None)?; + let native: &NativeScopeLookup = reports.lookup.get(); + let result = browse_scope_folders(reports.source(native), &request); + let diagnostics: Bound<'_, PyAny> = + native_field(&native_field(lookup, "index")?, "diagnostics")?; + Ok(browse_object(&classes, result, &diagnostics)?.unbind()) } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_explain.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_explain.rs index 86db324665..6045c41b99 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_explain.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_explain.rs @@ -1,19 +1,52 @@ //! `explain_scope_declaration`: one qualified declaration in an optional context. -use crate::scopes::_helpers::reports::scope_reports; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::reports::scope_report_objects::{ + diagnostics_object, explanation_object, read_declaration_target, read_optional_target, +}; +use crate::scopes::_helpers::reports::scope_reports::LookupReports; use crate::scopes::_helpers::reports::scope_reports::PyObject; +use crate::scopes::_helpers::reports::scope_reports::failure_error; +use crate::scopes::_helpers::reports::scope_reports::lookup_reports; +use crate::scopes::_helpers::reports::scope_reports::path_text; +use crate::scopes::_helpers::reports::scope_reports::raised; use crate::scopes::models::ExplainArguments; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::models::ScopeClasses; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_scopes::scope_report::main::explain_scope_declaration::explain_scope_declaration; +use sqlbuild_scopes::scope_report::models::ExplainRequest; /// `explain_scope_declaration`: one qualified declaration in an optional context. -#[inline] pub fn scope_explain( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: ExplainArguments<'_>, ) -> PyResult<(PyObject, PyObject)> { - scope_reports::scope_explain(py, lookup, request) + let ExplainArguments { + declaration, + target, + at, + directory, + } = request; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let reports: LookupReports = lookup_reports(lookup)?; + let request: ExplainRequest = ExplainRequest { + declaration: read_declaration_target(&classes, &declaration)?, + target: read_optional_target(&classes, target.as_ref())?, + at: path_text(at)?, + directory, + }; + let native: &NativeScopeLookup = reports.lookup.get(); + match explain_scope_declaration(reports.source(native), &request) { + Ok((explanation, diagnostics)) => { + let index_diagnostics: Bound<'_, PyAny> = + native_field(&native_field(lookup, "index")?, "diagnostics")?; + Ok(( + explanation_object(&classes, explanation)?.unbind(), + diagnostics_object(&classes, &index_diagnostics, raised(diagnostics))?.unbind(), + )) + } + Err(failure) => Err(failure_error(py, &failure)?), + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_list.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_list.rs index 7460b26b08..17aee73a7e 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_list.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_list.rs @@ -1,19 +1,42 @@ //! `list_scope_declarations`: declarations recursively beneath one definition folder. -use crate::scopes::_helpers::reports::scope_reports; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::reports::scope_report_objects::list_object; +use crate::scopes::_helpers::reports::scope_reports::LookupReports; use crate::scopes::_helpers::reports::scope_reports::PyObject; +use crate::scopes::_helpers::reports::scope_reports::failure_error; +use crate::scopes::_helpers::reports::scope_reports::folder_request; +use crate::scopes::_helpers::reports::scope_reports::lookup_reports; use crate::scopes::models::ListArguments; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::models::ScopeClasses; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_scopes::scope_report::main::list_scope_declarations::list_scope_declarations; +use sqlbuild_scopes::scope_report::models::FolderRequest; /// `list_scope_declarations`: declarations recursively beneath one definition folder. -#[inline] pub fn scope_list( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: ListArguments<'_>, ) -> PyResult { - scope_reports::scope_list(py, lookup, request) + let ListArguments { + folder, + filters, + target, + target_is_prospective, + } = request; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let reports: LookupReports = lookup_reports(lookup)?; + let request: FolderRequest = + folder_request(&classes, (folder, target, target_is_prospective), filters)?; + let native: &NativeScopeLookup = reports.lookup.get(); + match list_scope_declarations(reports.source(native), &request) { + Ok(result) => { + let diagnostics: Bound<'_, PyAny> = + native_field(&native_field(lookup, "index")?, "diagnostics")?; + Ok(list_object(&classes, result, &diagnostics)?.unbind()) + } + Err(failure) => Err(failure_error(py, &failure)?), + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_move_preview.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_move_preview.rs index 21dbee2b9e..7df2950f65 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_move_preview.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_move_preview.rs @@ -1,19 +1,48 @@ //! `preview_scope_move`: visibility deltas of a resource move, without mutating the lookup. -use crate::scopes::_helpers::reports::scope_reports; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::reports::scope_report_objects::{ + diagnostics_object, move_object, read_target, +}; +use crate::scopes::_helpers::reports::scope_reports::LookupReports; use crate::scopes::_helpers::reports::scope_reports::PyObject; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::reports::scope_reports::failure_error; +use crate::scopes::_helpers::reports::scope_reports::lookup_reports; +use crate::scopes::_helpers::reports::scope_reports::raised; +use crate::scopes::models::ScopeClasses; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_scopes::scope_report::main::preview_scope_move::preview_scope_move; +use sqlbuild_scopes::scope_report::models::MoveRequest; /// `preview_scope_move`: visibility deltas of a resource move, without mutating the lookup. -#[inline] +#[pyfunction] +#[pyo3(name = "preview_scope_move", signature = (*, lookup, resource, destination))] pub fn scope_move_preview( py: Python<'_>, lookup: &Bound<'_, PyAny>, resource: &Bound<'_, PyAny>, destination: &Bound<'_, PyAny>, ) -> PyResult<(Option, PyObject)> { - scope_reports::scope_move_preview(py, lookup, resource, destination) + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let reports: LookupReports = lookup_reports(lookup)?; + let request: MoveRequest = MoveRequest { + resource: read_target(&classes, resource)?, + destination: destination.str()?.extract()?, + }; + let native: &NativeScopeLookup = reports.lookup.get(); + match preview_scope_move(reports.source(native), &request) { + Ok((preview, diagnostics)) => { + let index_diagnostics: Bound<'_, PyAny> = + native_field(&native_field(lookup, "index")?, "diagnostics")?; + Ok(( + preview + .map(|preview| Ok::<_, PyErr>(move_object(&classes, preview)?.unbind())) + .transpose()?, + diagnostics_object(&classes, &index_diagnostics, raised(diagnostics))?.unbind(), + )) + } + Err(failure) => Err(failure_error(py, &failure)?), + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_rebound_usages.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_rebound_usages.rs index ac85c5fe37..c87288db12 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_rebound_usages.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_rebound_usages.rs @@ -1,17 +1,22 @@ //! Scoped declarations the resources at `path` use that resolve differently from `destination`. -use crate::scopes::_helpers::reports::scope_reports; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::reports::scope_reports::LookupReports; +use crate::scopes::_helpers::reports::scope_reports::failure_error; +use crate::scopes::_helpers::reports::scope_reports::lookup_reports; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_scopes::scope_report::main::rebound_scope_usages::rebound_scope_usages; /// Scoped declarations the resources at `path` use that resolve differently from `destination`. -#[inline] pub fn scope_rebound_usages( py: Python<'_>, lookup: &Bound<'_, PyAny>, paths: (&str, &str), ) -> PyResult>> { - scope_reports::scope_rebound_usages(py, lookup, paths) + let reports: LookupReports = lookup_reports(lookup)?; + let native: &NativeScopeLookup = reports.lookup.get(); + match rebound_scope_usages(reports.source(native), paths) { + Ok(rebound) => Ok(rebound), + Err(failure) => Err(failure_error(py, &failure)?), + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs index 5ad16340fa..a03b663ad1 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs @@ -1,19 +1,53 @@ //! `query_scope_report`: one existing target or prospective path; invalid paths raise. -use crate::scopes::_helpers::reports::scope_reports; +use crate::scopes::_helpers::index::scope_lookup::NativeScopeLookup; +use crate::scopes::_helpers::reports::scope_report_objects::{ + read_filters, read_optional_target, report_object, +}; +use crate::scopes::_helpers::reports::scope_reports::LookupReports; use crate::scopes::_helpers::reports::scope_reports::PyObject; +use crate::scopes::_helpers::reports::scope_reports::failure_error; +use crate::scopes::_helpers::reports::scope_reports::lookup_reports; +use crate::scopes::_helpers::reports::scope_reports::path_text; use crate::scopes::models::ReportArguments; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::models::ScopeClasses; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_scopes::scope_report::main::build_scope_report::build_scope_report; +use sqlbuild_scopes::scope_report::models::ReportRequest; /// `query_scope_report`: one existing target or prospective path; invalid paths raise. -#[inline] pub fn scope_report( py: Python<'_>, lookup: &Bound<'_, PyAny>, request: ReportArguments<'_>, ) -> PyResult { - scope_reports::scope_report(py, lookup, request) + let ReportArguments { + target, + at, + directory, + filters, + } = request; + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + let selected: Bound<'_, PyAny> = match filters.filter(|filters| !filters.is_none()) { + Some(filters) => filters, + None => classes.model(RecordId::ScopeReportFilters)?.call0()?, + }; + let reports: LookupReports = lookup_reports(lookup)?; + let request: ReportRequest = ReportRequest { + target: read_optional_target(&classes, target.as_ref())?, + at: path_text(at)?, + directory, + filters: read_filters(&selected)?, + }; + let native: &NativeScopeLookup = reports.lookup.get(); + match build_scope_report(reports.source(native), &request) { + Ok(report) => { + let diagnostics: Bound<'_, PyAny> = + native_field(&native_field(lookup, "index")?, "diagnostics")?; + Ok(report_object(&classes, report, (&selected, &diagnostics))?.unbind()) + } + Err(failure) => Err(failure_error(py, &failure)?), + } } diff --git a/crates/sqlbuild-python-front/src/scopes/main/reports/serialize_scope_report.rs b/crates/sqlbuild-python-front/src/scopes/main/reports/serialize_scope_report.rs index 91429ea6c6..6949771bb3 100644 --- a/crates/sqlbuild-python-front/src/scopes/main/reports/serialize_scope_report.rs +++ b/crates/sqlbuild-python-front/src/scopes/main/reports/serialize_scope_report.rs @@ -1,13 +1,25 @@ //! `serialize_scope_report`: deterministic schema-version-two ASCII JSON of one result. -use crate::scopes::_helpers::reports::scope_reports; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::scopes::_helpers::reports::scope_reports::scope_error; +use crate::scopes::_helpers::reports::scope_reports::serialized; +use crate::scopes::models::ScopeClasses; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::ErrorId; /// `serialize_scope_report`: deterministic schema-version-two ASCII JSON of one result. -#[inline] +#[pyfunction] +#[pyo3(signature = (*, report))] pub fn serialize_scope_report(py: Python<'_>, report: &Bound<'_, PyAny>) -> PyResult { - scope_reports::serialize_scope_report(py, report) + let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; + match serialized(&classes, report) { + Err(error) if error.is_instance_of::(py) => { + let message: String = error.value(py).str()?.extract()?; + let raised: PyErr = scope_error(py, ErrorId::Scope, message)?; + raised.set_cause(py, Some(error)); + Err(raised) + } + other => other, + } } diff --git a/crates/sqlbuild-python-front/src/sql_test_glue/_helpers/authored_ctes.rs b/crates/sqlbuild-python-front/src/sql_test_glue/_helpers/authored_ctes.rs index e6e78be935..40ff86c4c1 100644 --- a/crates/sqlbuild-python-front/src/sql_test_glue/_helpers/authored_ctes.rs +++ b/crates/sqlbuild-python-front/src/sql_test_glue/_helpers/authored_ctes.rs @@ -1,10 +1,7 @@ //! Reject authored SQL tests and scenarios whose CTE names cannot be read, at the authored text. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::main::text_position::text_position; use sqlbuild_analysis::compiled_project::models::TextPosition; use sqlbuild_analysis::compiler::main::sql_test_classification::extract_sql_tests; @@ -12,31 +9,14 @@ use sqlbuild_analysis::compiler::models::{SqlTestExtractionFailure, SqlTestReque use sqlbuild_analysis::sql_test_attachment::main::authored_test_offset::authored_test_offset; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::sqltext::models::LexicalSyntaxInput; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; const MODEL_MODE: &str = "model"; -/// Extract fixture bodies from authored test or scenario CTEs in authored order. -pub(crate) fn prefixed_ctes<'py>( - ctes: &Bound<'py, PyAny>, - prefix: &str, -) -> PyResult> { - let result = PyDict::new(ctes.py()); - for cte in ctes.try_iter()? { - let cte = cte?; - let name: String = native_field(&cte, "name")?.extract()?; - if let Some(name) = name.strip_prefix(prefix) { - result.set_item(name, native_field(&cte, "sql_body")?)?; - } - } - Ok(result) -} - /// One authored SQL body to read: its text, file contents, relative path and body span. -struct AuthoredBody { +pub(crate) struct AuthoredBody { sql: String, contents: String, relative_path: String, @@ -46,7 +26,7 @@ struct AuthoredBody { } impl AuthoredBody { - fn read( + pub(crate) fn read( owner: &Bound<'_, PyAny>, sql: &Bound<'_, PyAny>, file: &Bound<'_, PyAny>, @@ -89,7 +69,7 @@ fn located_error(py: Python<'_>, failure: &SqlTestExtractionFailure, body: &Auth } /// Raise the located error of the first body whose authored CTEs cannot be read. -fn check_bodies( +pub(crate) fn check_bodies( py: Python<'_>, bodies: &[AuthoredBody], scenario: bool, @@ -114,41 +94,3 @@ fn check_bodies( Err(failure) => Err(located_error(py, &failure, &bodies[failure.index])), } } - -/// Reject the first authored test block, then scenario, whose CTE names cannot be read. -#[pyfunction] -pub(crate) fn validate_authored_sql_test_ctes( - py: Python<'_>, - test_files: Bound<'_, PyAny>, - scenario_files: Bound<'_, PyAny>, - syntax: LexicalSyntaxInput, -) -> Result<(), PyErr> { - let syntax: LexicalSyntax = syntax.into(); - let mut blocks: Vec = Vec::new(); - for test_file in test_files.try_iter()? { - let test_file: Bound<'_, PyAny> = test_file?; - for block in native_field(&test_file, "blocks")?.try_iter()? { - let block: Bound<'_, PyAny> = block?; - blocks.push(AuthoredBody::read( - &block, - &native_field(&block, "sql_body")?, - &test_file, - )?); - } - } - check_bodies(py, &blocks, false, &syntax)?; - let mut scenarios: Vec = Vec::new(); - for scenario_file in scenario_files.try_iter()? { - let scenario_file: Bound<'_, PyAny> = scenario_file?; - scenarios.push(AuthoredBody::read( - &scenario_file, - &native_field(&scenario_file, "sql_body")?, - &scenario_file, - )?); - } - check_bodies(py, &scenarios, true, &syntax) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(validate_authored_sql_test_ctes, module)?) -} diff --git a/crates/sqlbuild-python-front/src/sql_test_glue/main/prefixed_ctes.rs b/crates/sqlbuild-python-front/src/sql_test_glue/main/prefixed_ctes.rs index ee70b193f8..2983897917 100644 --- a/crates/sqlbuild-python-front/src/sql_test_glue/main/prefixed_ctes.rs +++ b/crates/sqlbuild-python-front/src/sql_test_glue/main/prefixed_ctes.rs @@ -1,13 +1,18 @@ //! Extract fixture bodies from authored test or scenario CTEs in authored order. -use crate::sql_test_glue::_helpers::authored_ctes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Extract fixture bodies from authored test or scenario CTEs in authored order. -#[inline] pub fn prefixed_ctes<'py>(ctes: &Bound<'py, PyAny>, prefix: &str) -> PyResult> { - authored_ctes::prefixed_ctes(ctes, prefix) + let result = PyDict::new(ctes.py()); + for cte in ctes.try_iter()? { + let cte = cte?; + let name: String = native_field(&cte, "name")?.extract()?; + if let Some(name) = name.strip_prefix(prefix) { + result.set_item(name, native_field(&cte, "sql_body")?)?; + } + } + Ok(result) } diff --git a/crates/sqlbuild-python-front/src/sql_test_glue/main/register_authored_ctes.rs b/crates/sqlbuild-python-front/src/sql_test_glue/main/register_authored_ctes.rs index 06b795b1ca..f366bef1ed 100644 --- a/crates/sqlbuild-python-front/src/sql_test_glue/main/register_authored_ctes.rs +++ b/crates/sqlbuild-python-front/src/sql_test_glue/main/register_authored_ctes.rs @@ -1,12 +1,10 @@ //! Register the authored ctes functions and classes. -use crate::sql_test_glue::_helpers::authored_ctes; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::sql_test_glue::main::validate_authored_sql_test_ctes::validate_authored_sql_test_ctes; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the authored ctes functions and classes. -#[inline] pub fn register_authored_ctes(module: &Bound<'_, PyModule>) -> PyResult<()> { - authored_ctes::register(module) + module.add_function(wrap_pyfunction!(validate_authored_sql_test_ctes, module)?) } diff --git a/crates/sqlbuild-python-front/src/sql_test_glue/main/validate_authored_sql_test_ctes.rs b/crates/sqlbuild-python-front/src/sql_test_glue/main/validate_authored_sql_test_ctes.rs index 4fb8a7c492..ec1f1dd0e2 100644 --- a/crates/sqlbuild-python-front/src/sql_test_glue/main/validate_authored_sql_test_ctes.rs +++ b/crates/sqlbuild-python-front/src/sql_test_glue/main/validate_authored_sql_test_ctes.rs @@ -1,19 +1,43 @@ //! Reject the first authored test block, then scenario, whose CTE names cannot be read. -use crate::sql_test_glue::_helpers::authored_ctes; +use crate::sql_test_glue::_helpers::authored_ctes::AuthoredBody; +use crate::sql_test_glue::_helpers::authored_ctes::check_bodies; use crate::sqltext::models::LexicalSyntaxInput; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyErr; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// Reject the first authored test block, then scenario, whose CTE names cannot be read. -#[inline] +#[pyfunction] pub fn validate_authored_sql_test_ctes( py: Python<'_>, test_files: Bound<'_, PyAny>, scenario_files: Bound<'_, PyAny>, syntax: LexicalSyntaxInput, ) -> Result<(), PyErr> { - authored_ctes::validate_authored_sql_test_ctes(py, test_files, scenario_files, syntax) + let syntax: LexicalSyntax = syntax.into(); + let mut blocks: Vec = Vec::new(); + for test_file in test_files.try_iter()? { + let test_file: Bound<'_, PyAny> = test_file?; + for block in native_field(&test_file, "blocks")?.try_iter()? { + let block: Bound<'_, PyAny> = block?; + blocks.push(AuthoredBody::read( + &block, + &native_field(&block, "sql_body")?, + &test_file, + )?); + } + } + check_bodies(py, &blocks, false, &syntax)?; + let mut scenarios: Vec = Vec::new(); + for scenario_file in scenario_files.try_iter()? { + let scenario_file: Bound<'_, PyAny> = scenario_file?; + scenarios.push(AuthoredBody::read( + &scenario_file, + &native_field(&scenario_file, "sql_body")?, + &scenario_file, + )?); + } + check_bodies(py, &scenarios, true, &syntax) } diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/authored_values.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/authored_values.rs index d0fbd143e8..cc9c278cba 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/authored_values.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/authored_values.rs @@ -1,13 +1,15 @@ //! Authored header values as the Python header objects: scalars, sets, constants and hooks. +use crate::sqltext::main::authored_value_to_python::authored_value_to_python; +use crate::sqltext::main::map_to_python::map_to_python; use pyo3::prelude::{IntoPyObject, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; use sqlbuild_sqltext::compiler::models::{AuthoredValue, HeaderWord}; -fn word_to_python(py: Python<'_>, word: &str) -> PyResult> { +pub(crate) fn word_to_python(py: Python<'_>, word: &str) -> PyResult> { match header_word_value(word) { HeaderWord::Bool(value) => Ok(value.into_pyobject(py)?.to_owned().unbind().into_any()), HeaderWord::Null => Ok(py.None()), @@ -24,7 +26,7 @@ fn word_to_python(py: Python<'_>, word: &str) -> PyResult> { } } -fn instance( +pub(crate) fn instance( py: Python<'_>, class: RecordId, arguments: &pyo3::Bound<'_, PyDict>, @@ -32,7 +34,7 @@ fn instance( Ok(new_record(py, class, (), Some(arguments))?.unbind()) } -fn values_tuple(py: Python<'_>, values: &[AuthoredValue]) -> PyResult> { +pub(crate) fn values_tuple(py: Python<'_>, values: &[AuthoredValue]) -> PyResult> { let projected = values .iter() .map(|item| authored_value_to_python(py, item)) @@ -40,7 +42,10 @@ fn values_tuple(py: Python<'_>, values: &[AuthoredValue]) -> PyResult> Ok(PyTuple::new(py, projected)?.unbind().into_any()) } -fn constant_call(py: Python<'_>, values: &[(String, AuthoredValue)]) -> PyResult> { +pub(crate) fn constant_call( + py: Python<'_>, + values: &[(String, AuthoredValue)], +) -> PyResult> { let arguments = values .iter() .map(|(key, value)| { @@ -53,59 +58,7 @@ fn constant_call(py: Python<'_>, values: &[(String, AuthoredValue)]) -> PyResult instance(py, RecordId::AuthoredSqlValueCall, &keywords) } -/// One authored value as the Python header object discovery hands to config builds. -pub(crate) fn authored_value_to_python( - py: Python<'_>, - value: &AuthoredValue, -) -> PyResult> { - match value { - AuthoredValue::Null => Ok(py.None()), - AuthoredValue::Boolean(value) => { - Ok(value.into_pyobject(py)?.to_owned().unbind().into_any()) - } - AuthoredValue::BareWord(value) => word_to_python(py, value), - AuthoredValue::String(value) => Ok(value.into_pyobject(py)?.unbind().into_any()), - AuthoredValue::List(values) => { - let projected = values - .iter() - .map(|item| authored_value_to_python(py, item)) - .collect::>>()?; - Ok(PyList::new(py, projected)?.unbind().into_any()) - } - AuthoredValue::Map(values) => map_to_python(py, values), - AuthoredValue::Set(values) => { - let keywords = PyDict::new(py); - keywords.set_item("values", values_tuple(py, values)?)?; - instance(py, RecordId::AuthoredSqlSet, &keywords) - } - AuthoredValue::Tuple(values) => values_tuple(py, values), - AuthoredValue::TypedConstant(values) => constant_call(py, values), - AuthoredValue::InlineSqlHook(statement) => { - let keywords = PyDict::new(py); - keywords.set_item("statement", statement)?; - instance(py, RecordId::SqlHookEntry, &keywords) - } - AuthoredValue::NamedSqlHook(name, kwargs) => { - hook(py, RecordId::NamedSqlHookEntry, name, kwargs) - } - AuthoredValue::PythonHook(name, kwargs) => { - hook(py, RecordId::PythonHookEntry, name, kwargs) - } - } -} - -pub(crate) fn map_to_python( - py: Python<'_>, - values: &[(String, AuthoredValue)], -) -> PyResult> { - let result = PyDict::new(py); - for (key, value) in values { - result.set_item(key, authored_value_to_python(py, value)?)?; - } - Ok(result.unbind().into_any()) -} - -fn hook( +pub(crate) fn hook( py: Python<'_>, class: RecordId, name: &str, @@ -116,12 +69,3 @@ fn hook( keywords.set_item("kwargs", map_to_python(py, kwargs)?)?; instance(py, class, &keywords) } - -pub(crate) fn optional_authored_value_to_python( - py: Python<'_>, - value: Option<&AuthoredValue>, -) -> PyResult>> { - value - .map(|item| authored_value_to_python(py, item)) - .transpose() -} diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/code_scanner.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/code_scanner.rs deleted file mode 100644 index 948a471284..0000000000 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/code_scanner.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Comment, quoted-text and parenthesis boundaries at Python string offsets of one SQL text. - -use crate::sqltext::models::SqlCodeScanner; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/ddl_effects.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/ddl_effects.rs index c0ec6dba04..d5c71d554b 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/ddl_effects.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/ddl_effects.rs @@ -1,9 +1,9 @@ //! Executed-statement effects on the executor's cached relation metadata. -use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::pyfunction; use pyo3::sync::PyOnceLock; use pyo3::types::{PyBool, PyFrozenSet}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_sqltext::ddl_effects::main::qualified_relation_name_key::qualified_relation_name_key as native_qualified_relation_name_key; use sqlbuild_sqltext::ddl_effects::main::relation_name_key::relation_name_key as native_relation_name_key; @@ -20,7 +20,10 @@ static EFFECT_CLASS: PyOnceLock> = PyOnceLock::new(); /// Which cached relation names `sql` may have changed, as a `StatementMetadataEffect`. #[pyfunction] #[pyo3(signature = (sql, /))] -fn statement_metadata_effect<'py>(py: Python<'py>, sql: &str) -> PyResult> { +pub(crate) fn statement_metadata_effect<'py>( + py: Python<'py>, + sql: &str, +) -> PyResult> { let effect: StatementMetadataEffect = py.detach(|| native_statement_metadata_effect(sql)); let class: &Bound<'py, PyAny> = EFFECT_CLASS .get_or_try_init(py, || host_entry(py, EFFECT_MODEL).map(Bound::unbind))? @@ -51,20 +54,13 @@ fn statement_metadata_effect<'py>(py: Python<'py>, sql: &str) -> PyResult String { +pub(crate) fn relation_name_key(name: &str) -> String { native_relation_name_key(name) } /// The key for the last part of a qualified name, or `None` when it cannot be parsed. #[pyfunction] #[pyo3(signature = (qualified, /))] -fn qualified_relation_name_key(qualified: &str) -> Option { +pub(crate) fn qualified_relation_name_key(qualified: &str) -> Option { native_qualified_relation_name_key(qualified) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(statement_metadata_effect, module)?)?; - module.add_function(wrap_pyfunction!(relation_name_key, module)?)?; - module.add_function(wrap_pyfunction!(qualified_relation_name_key, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/declaration_references.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/declaration_references.rs index a31739eb01..d3f0ff8da0 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/declaration_references.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/declaration_references.rs @@ -1,9 +1,7 @@ //! `@enum`/`@const` reference offsets scanned natively for the model loop. use crate::sqltext::models::ScannedReference; -use sqlbuild_analysis::sql_macros::constants::SQL_CALL_PREFIX; use sqlbuild_analysis::sql_macros::main::sql_macro_calls::sql_macro_calls; -use sqlbuild_sqltext::compiler::main::declaration_references::scan_declaration_references; use sqlbuild_sqltext::compiler::models::{ DeclarationReference, DeclarationReferenceKind, DeclarationReferenceScan, DeclarationReferenceStop, @@ -16,19 +14,12 @@ const SQL_MACRO_REFERENCE_KIND: u8 = 2; /// References and stop code: unclosed quote 0, block comment 1, invalid enum 2, constant 3. pub(crate) type ScanRow = (Vec, Option); -/// One SQL string's references and stop code, scanned without walking past malformed ones. -pub(crate) fn declaration_scan(sql: &str, syntax: Option<&LexicalSyntax>) -> ScanRow { - let scanned: Vec = - scan_declaration_references(&[sql.to_owned()], false, syntax); - let row: ScanRow = scanned.into_iter().next().map(scan_row).unwrap_or_default(); - if !sql.contains(SQL_CALL_PREFIX) { - return row; - } - with_sql_macro_calls(sql, row, syntax) -} - /// The scan with each outer SQL macro call as one kind-2 reference, replacing those inside it. -fn with_sql_macro_calls(sql: &str, row: ScanRow, syntax: Option<&LexicalSyntax>) -> ScanRow { +pub(crate) fn with_sql_macro_calls( + sql: &str, + row: ScanRow, + syntax: Option<&LexicalSyntax>, +) -> ScanRow { let (references, stop) = row; let Ok(calls) = sql_macro_calls(sql, syntax) else { return (references, stop); @@ -66,7 +57,7 @@ fn with_sql_macro_calls(sql: &str, row: ScanRow, syntax: Option<&LexicalSyntax>) (merged, stop) } -fn scan_row(scan: DeclarationReferenceScan) -> ScanRow { +pub(crate) fn scan_row(scan: DeclarationReferenceScan) -> ScanRow { ( scan.references.into_iter().map(reference_row).collect(), scan.stop.map(|stop| match stop { diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/lexical_syntax.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/lexical_syntax.rs index 200ce9e1eb..1881fce178 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/lexical_syntax.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/lexical_syntax.rs @@ -1,14 +1,8 @@ //! One adapter's `SqlLexicalSyntax`, read from a Python mapping. use crate::sqltext::models::LexicalSyntaxInput; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyFrozenSet; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; - impl From for LexicalSyntax { fn from(input: LexicalSyntaxInput) -> Self { Self { @@ -22,63 +16,3 @@ impl From for LexicalSyntax { } } } - -/// The sorted texts of a set of strings. -pub(crate) fn sorted_texts(values: &Bound<'_, PyAny>) -> PyResult> { - let mut texts: Vec = values - .try_iter()? - .map(|value| value?.extract::()) - .collect::>()?; - texts.sort_unstable(); - Ok(texts) -} - -/// The lexical rules a `SqlLexicalSyntax` record holds. -pub(crate) fn record_syntax(instance: &Bound<'_, PyAny>) -> PyResult { - let flag = |name: &str| -> PyResult { native_field(instance, name)?.is_truthy() }; - Ok(LexicalSyntax { - backslash_escape_quotes: sorted_texts(&native_field(instance, "backslash_escape_quotes")?)?, - escape_string_prefix: flag("escape_string_prefix")?, - raw_string_prefix: flag("raw_string_prefix")?, - triple_quoted_strings: flag("triple_quoted_strings")?, - nested_block_comments: flag("nested_block_comments")?, - line_comment_prefixes: sorted_texts(&native_field(instance, "line_comment_prefixes")?)?, - bracket_identifiers: false, - }) -} - -/// The lexical rules a `SqlLexicalSyntax` record holds, or `None` for `None`. -pub(crate) fn optional_record_syntax(value: &Bound<'_, PyAny>) -> PyResult> { - if value.is_none() { - return Ok(None); - } - record_syntax(value).map(Some) -} - -/// A `SqlLexicalSyntax` record holding `syntax`. -pub(crate) fn syntax_record<'py>( - py: Python<'py>, - syntax: &LexicalSyntax, -) -> PyResult> { - let flag = |value: bool| -> PyResult> { - Ok(pyo3::types::PyBool::new(py, value).to_owned().into_any()) - }; - new_record_with( - py, - RecordId::SqlLexicalSyntax, - &[ - ( - "backslash_escape_quotes", - PyFrozenSet::new(py, &syntax.backslash_escape_quotes)?.into_any(), - ), - ("escape_string_prefix", flag(syntax.escape_string_prefix)?), - ("raw_string_prefix", flag(syntax.raw_string_prefix)?), - ("triple_quoted_strings", flag(syntax.triple_quoted_strings)?), - ("nested_block_comments", flag(syntax.nested_block_comments)?), - ( - "line_comment_prefixes", - PyFrozenSet::new(py, &syntax.line_comment_prefixes)?.into_any(), - ), - ], - ) -} diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/mod.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/mod.rs index 4a2ee8426a..97b4ef2d10 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/mod.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod authored_values; -pub(crate) mod code_scanner; pub(crate) mod ddl_effects; pub(crate) mod declaration_references; pub(crate) mod lexical_syntax; diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/model_headers.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/model_headers.rs index f8c8ecd8c9..56ec8fd04b 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/model_headers.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/model_headers.rs @@ -1,71 +1,15 @@ //! Match, tokenize and parse model headers natively. +use crate::sqltext::main::location_rows::location_rows; use crate::sqltext::models::ColumnLocation; -use crate::sqltext::types::ParsedModelHeader; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::PyResult; +use pyo3::pyfunction; -use crate::sqltext::_helpers::authored_values::optional_authored_value_to_python; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::boundary::main::value_error::value_error; -use sqlbuild_python_base::types::CompilerDetach; - -#[pyfunction] -pub(crate) fn parse_model_headers( - py: Python<'_>, - headers: Vec, -) -> PyResult> { - let parsed = py - .compiler_detach(|| { - sqlbuild_sqltext::compiler::main::model_header_parsing::parse_batch(&headers) - }) - .map_err(value_error)?; - parsed - .into_iter() - .map(|(value, offsets, error)| { - Ok(( - optional_authored_value_to_python(py, value.as_ref())?, - offsets, - error, - )) - }) - .collect() -} - -pub(crate) fn location_rows( - locations: Vec<(String, sqlbuild_sqltext::compiler::models::TextLocation)>, -) -> Vec { - locations - .into_iter() - .map(|(name, location)| ColumnLocation { - name, - line: location.line, - column: location.column, - end_line: location.end_line, - end_column: location.end_column, - }) - .collect() -} - -/// Simple top-level SELECT output names and their authored locations. -#[pyfunction] -pub(crate) fn model_output_columns( - contents: &str, - implicit_aliases: bool, -) -> PyResult> { - compiler_guard(|| { - Ok(location_rows( - sqlbuild_sqltext::compiler::main::model_output_columns::model_output_columns( - contents, - implicit_aliases, - ), - )) - }) -} /// The authored locations of a header's column names, or the header's syntax error. #[pyfunction] -fn model_header_column_locations( +pub(crate) fn model_header_column_locations( contents: &str, header: &str, header_start: usize, @@ -83,39 +27,3 @@ fn model_header_column_locations( ) }) } - -/// A header syntax error's detail, help and file line (when `header_line` is known). -pub(crate) fn header_failure( - error: &str, - header: &str, - header_line: Option, -) -> (String, Option, Option) { - let nesting = sqlbuild_sqltext::compiler::main::model_header_errors::header_nesting_failure( - error, - header, - header_line.unwrap_or(1), - ); - if let Some(quote) = sqlbuild_sqltext::header_strings::main::quote_failure::quote_failure( - error, - header, - (header_line.unwrap_or(1), 1), - ) { - return ( - quote.message, - Some(quote.help), - header_line.map(|_| quote.line), - ); - } - match (nesting, header_line) { - (Some(nesting), Some(_)) => (nesting.message, Some(nesting.help), Some(nesting.line)), - (Some(nesting), None) => (error.to_owned(), Some(nesting.help), None), - (None, _) => (error.to_owned(), None, None), - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_model_headers, module)?)?; - module.add_function(wrap_pyfunction!(model_output_columns, module)?)?; - module.add_function(wrap_pyfunction!(model_header_column_locations, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/placeholders.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/placeholders.rs index 5ae54c3196..d44a0feb5e 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/placeholders.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/placeholders.rs @@ -1,7 +1,7 @@ //! Substitute `@@@name` placeholder defaults for SQL analysis. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; @@ -9,7 +9,7 @@ use sqlbuild_python_base::types::CompilerDetach; /// Python's `substitute_placeholder_defaults`: each `@@@name` becomes its default. #[pyfunction] -fn substitute_placeholder_defaults( +pub(crate) fn substitute_placeholder_defaults( py: Python<'_>, sql: &str, placeholders: Vec<(String, String)>, @@ -17,8 +17,3 @@ fn substitute_placeholder_defaults( py.compiler_detach(|| Ok(placeholder_defaults(sql, &placeholders))) .map_err(compiler_error) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(substitute_placeholder_defaults, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs index c45eecade0..1caa7ef112 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs @@ -1,10 +1,8 @@ //! Logical SQL references extracted natively under one adapter's lexical rules. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyTuple; -use pyo3::{IntoPyObject, IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::{IntoPyObject, IntoPyObjectExt, pyclass, pyfunction, pymethods}; use sqlbuild_python_base::positional_record; use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; use sqlbuild_sqltext::sql_references::models::{ @@ -14,16 +12,14 @@ use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::sqltext::models::LexicalSyntaxInput; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::compile_context::main::diagnostic_collector::report_compile_diagnostic::report_compile_diagnostic; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; +pub(crate) const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; positional_record! { into; @@ -113,12 +109,15 @@ fn invalid_call_row(call: InvalidReferenceCall) -> InvalidCallRow { } } -fn text<'py>(py: Python<'py>, value: impl IntoPyObject<'py>) -> PyResult> { +pub(crate) fn text<'py>( + py: Python<'py>, + value: impl IntoPyObject<'py>, +) -> PyResult> { value.into_bound_py_any(py) } /// The references and rejected calls `sql` holds as Python objects; a scan error raises. -fn scanned<'py>( +pub(crate) fn scanned<'py>( py: Python<'py>, sql: &str, syntax: &Bound<'py, PyAny>, @@ -182,7 +181,7 @@ fn scanned<'py>( /// Valid references and rejected calls; a scan error raises unlocated. #[pyfunction] #[pyo3(signature = (*, sql, syntax))] -fn scan_sql_reference_calls<'py>( +pub(crate) fn scan_sql_reference_calls<'py>( sql: &str, syntax: &Bound<'py, PyAny>, ) -> PyResult> { @@ -197,47 +196,3 @@ fn scan_sql_reference_calls<'py>( ], ) } - -/// Logical SQL refs outside comments and quotes, reporting rejected calls as P012. -#[pyfunction(name = "extract_sql_references")] -#[pyo3(signature = (*, sql, syntax))] -pub(crate) fn extract_references<'py>( - sql: &str, - syntax: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = syntax.py(); - let (references, invalid_calls) = scanned(py, sql, syntax)?; - for call in invalid_calls { - let diagnostic: Bound<'py, PyAny> = new_record_with( - py, - RecordId::CompilerDiagnostic, - &[ - ( - "phase", - enum_member(py, RecordId::DiagnosticPhase, "COMPILE")?, - ), - ( - "severity", - enum_member(py, RecordId::DiagnosticSeverity, "ERROR")?, - ), - ("code", text(py, REFERENCE_CALL_SYNTAX_CODE)?), - ("message", native_field(&call, "message")?), - ("help", native_field(&call, "help")?), - ], - )?; - report_compile_diagnostic( - (REFERENCE_CALL_SYNTAX_CODE, "", native_field(&call, "call")?) - .into_pyobject(py)? - .as_any(), - &diagnostic, - )?; - } - PyTuple::new(py, references) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(scan_sql_reference_calls, module)?)?; - module.add_function(wrap_pyfunction!(extract_references, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs index 1e37cd63c7..4aa3789808 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs @@ -1,25 +1,23 @@ //! Interpolate `@@` tokens in authored SQL and extract static SQL references. -use crate::sqltext::models::InterpolationRow; use crate::sqltext::models::InterpolationSources; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use std::cell::RefCell; use pyo3::PyErr; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyString, PyStringMethods}; -use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; use sqlbuild_sqltext::compiler::models::InterpolationRead; use sqlbuild_sqltext::compiler::types::InterpolationHost; const ENVIRONMENT_READ: &str = "env"; const CONTEXT_READ: &str = "ctx"; -struct PythonInterpolationHost<'py> { - sources: InterpolationSources<'py>, +pub(crate) struct PythonInterpolationHost<'py> { + pub(crate) sources: InterpolationSources<'py>, /// A Python exception other than a rendering `ValueError`, raised after the scan. - raised: RefCell>, + pub(crate) raised: RefCell>, } impl PythonInterpolationHost<'_> { @@ -132,63 +130,10 @@ fn key_text(key: &Bound<'_, PyAny>) -> String { key.str().map(|text| text.to_string()).unwrap_or_default() } -/// Each `(sql, file path)` interpolated in order; non-rendering Python failures raise. -pub(crate) fn interpolation_batch( - sources: InterpolationSources<'_>, - sqls: &[(String, String)], -) -> PyResult> { - let host = PythonInterpolationHost { - sources, - raised: RefCell::new(None), - }; - let mut rows = Vec::with_capacity(sqls.len()); - for (sql, file_path) in sqls { - let row = match interpolate_sql(&host, sql, file_path) { - Ok(result) => InterpolationRow { - sql: result.sql, - spans: result.spans, - reads: read_rows(result.reads), - error: None, - }, - Err(failure) => InterpolationRow { - sql: None, - spans: Vec::new(), - reads: read_rows(failure.reads), - error: Some(failure.message), - }, - }; - if let Some(error) = host.raised.borrow_mut().take() { - return Err(error); - } - rows.push(row); - } - Ok(rows) -} - /// `(sql or None when unchanged, reads, error message or None)` for one SQL text. pub(crate) type InterpolatedRow = (Option, Vec<(&'static str, String)>, Option); -/// One SQL text interpolated: the SQL (`None` when unchanged), its reads, and its error. -pub(crate) fn interpolated( - sources: InterpolationSources<'_>, - sql: &str, - file_path: &str, -) -> PyResult { - let host = PythonInterpolationHost { - sources, - raised: RefCell::new(None), - }; - let row = match interpolate_sql(&host, sql, file_path) { - Ok(result) => (result.sql, read_rows(result.reads), None), - Err(failure) => (None, read_rows(failure.reads), Some(failure.message)), - }; - if let Some(error) = host.raised.borrow_mut().take() { - return Err(error); - } - Ok(row) -} - -fn read_rows(reads: Vec) -> Vec<(&'static str, String)> { +pub(crate) fn read_rows(reads: Vec) -> Vec<(&'static str, String)> { reads .into_iter() .map(|read| match read { diff --git a/crates/sqlbuild-python-front/src/sqltext/main/authored_value_to_python.rs b/crates/sqlbuild-python-front/src/sqltext/main/authored_value_to_python.rs index 4b26dde9b3..7b51d0d16d 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/authored_value_to_python.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/authored_value_to_python.rs @@ -1,14 +1,50 @@ //! One authored value as the Python header object discovery hands to config builds. -use crate::sqltext::_helpers::authored_values; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::sqltext::_helpers::authored_values::constant_call; +use crate::sqltext::_helpers::authored_values::hook; +use crate::sqltext::_helpers::authored_values::instance; +use crate::sqltext::_helpers::authored_values::values_tuple; +use crate::sqltext::_helpers::authored_values::word_to_python; +use crate::sqltext::main::map_to_python::map_to_python; +use pyo3::prelude::{IntoPyObject, Py, PyAny, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList}; +use sqlbuild_python_base::records::types::RecordId; use sqlbuild_sqltext::compiler::models::AuthoredValue; /// One authored value as the Python header object discovery hands to config builds. -#[inline] pub fn authored_value_to_python(py: Python<'_>, value: &AuthoredValue) -> PyResult> { - authored_values::authored_value_to_python(py, value) + match value { + AuthoredValue::Null => Ok(py.None()), + AuthoredValue::Boolean(value) => { + Ok(value.into_pyobject(py)?.to_owned().unbind().into_any()) + } + AuthoredValue::BareWord(value) => word_to_python(py, value), + AuthoredValue::String(value) => Ok(value.into_pyobject(py)?.unbind().into_any()), + AuthoredValue::List(values) => { + let projected = values + .iter() + .map(|item| authored_value_to_python(py, item)) + .collect::>>()?; + Ok(PyList::new(py, projected)?.unbind().into_any()) + } + AuthoredValue::Map(values) => map_to_python(py, values), + AuthoredValue::Set(values) => { + let keywords = PyDict::new(py); + keywords.set_item("values", values_tuple(py, values)?)?; + instance(py, RecordId::AuthoredSqlSet, &keywords) + } + AuthoredValue::Tuple(values) => values_tuple(py, values), + AuthoredValue::TypedConstant(values) => constant_call(py, values), + AuthoredValue::InlineSqlHook(statement) => { + let keywords = PyDict::new(py); + keywords.set_item("statement", statement)?; + instance(py, RecordId::SqlHookEntry, &keywords) + } + AuthoredValue::NamedSqlHook(name, kwargs) => { + hook(py, RecordId::NamedSqlHookEntry, name, kwargs) + } + AuthoredValue::PythonHook(name, kwargs) => { + hook(py, RecordId::PythonHookEntry, name, kwargs) + } + } } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/declaration_scan.rs b/crates/sqlbuild-python-front/src/sqltext/main/declaration_scan.rs index 78b1018b7c..5d5685a429 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/declaration_scan.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/declaration_scan.rs @@ -1,11 +1,20 @@ //! One SQL string's references and stop code, scanned without walking past malformed ones. -use crate::sqltext::_helpers::declaration_references; use crate::sqltext::_helpers::declaration_references::ScanRow; +use crate::sqltext::_helpers::declaration_references::scan_row; +use crate::sqltext::_helpers::declaration_references::with_sql_macro_calls; +use sqlbuild_analysis::sql_macros::constants::SQL_CALL_PREFIX; +use sqlbuild_sqltext::compiler::main::declaration_references::scan_declaration_references; +use sqlbuild_sqltext::compiler::models::DeclarationReferenceScan; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// One SQL string's references and stop code, scanned without walking past malformed ones. -#[inline] pub fn declaration_scan(sql: &str, syntax: Option<&LexicalSyntax>) -> ScanRow { - declaration_references::declaration_scan(sql, syntax) + let scanned: Vec = + scan_declaration_references(&[sql.to_owned()], false, syntax); + let row: ScanRow = scanned.into_iter().next().map(scan_row).unwrap_or_default(); + if !sql.contains(SQL_CALL_PREFIX) { + return row; + } + with_sql_macro_calls(sql, row, syntax) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/extract_references.rs b/crates/sqlbuild-python-front/src/sqltext/main/extract_references.rs index dd8e5d84b4..d735a9afa7 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/extract_references.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/extract_references.rs @@ -1,16 +1,50 @@ //! Logical SQL refs outside comments and quotes, reporting rejected calls as P012. -use crate::sqltext::_helpers::sql_references; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::sqltext::_helpers::sql_references::REFERENCE_CALL_SYNTAX_CODE; +use crate::sqltext::_helpers::sql_references::scanned; +use crate::sqltext::_helpers::sql_references::text; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyTuple; +use pyo3::{IntoPyObject, pyfunction}; +use sqlbuild_python_base::compile_context::main::diagnostic_collector::report_compile_diagnostic::report_compile_diagnostic; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; /// Logical SQL refs outside comments and quotes, reporting rejected calls as P012. -#[inline] +#[pyfunction(name = "extract_sql_references")] +#[pyo3(signature = (*, sql, syntax))] pub fn extract_references<'py>( sql: &str, syntax: &Bound<'py, PyAny>, ) -> PyResult> { - sql_references::extract_references(sql, syntax) + let py: Python<'py> = syntax.py(); + let (references, invalid_calls) = scanned(py, sql, syntax)?; + for call in invalid_calls { + let diagnostic: Bound<'py, PyAny> = new_record_with( + py, + RecordId::CompilerDiagnostic, + &[ + ( + "phase", + enum_member(py, RecordId::DiagnosticPhase, "COMPILE")?, + ), + ( + "severity", + enum_member(py, RecordId::DiagnosticSeverity, "ERROR")?, + ), + ("code", text(py, REFERENCE_CALL_SYNTAX_CODE)?), + ("message", native_field(&call, "message")?), + ("help", native_field(&call, "help")?), + ], + )?; + report_compile_diagnostic( + (REFERENCE_CALL_SYNTAX_CODE, "", native_field(&call, "call")?) + .into_pyobject(py)? + .as_any(), + &diagnostic, + )?; + } + PyTuple::new(py, references) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/header_failure.rs b/crates/sqlbuild-python-front/src/sqltext/main/header_failure.rs index 3a0a42ef4a..6b50d2fab8 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/header_failure.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/header_failure.rs @@ -1,13 +1,30 @@ //! A header syntax error's detail, help and file line (when `header_line` is known). -use crate::sqltext::_helpers::model_headers; - /// A header syntax error's detail, help and file line (when `header_line` is known). -#[inline] pub fn header_failure( error: &str, header: &str, header_line: Option, ) -> (String, Option, Option) { - model_headers::header_failure(error, header, header_line) + let nesting = sqlbuild_sqltext::compiler::main::model_header_errors::header_nesting_failure( + error, + header, + header_line.unwrap_or(1), + ); + if let Some(quote) = sqlbuild_sqltext::header_strings::main::quote_failure::quote_failure( + error, + header, + (header_line.unwrap_or(1), 1), + ) { + return ( + quote.message, + Some(quote.help), + header_line.map(|_| quote.line), + ); + } + match (nesting, header_line) { + (Some(nesting), Some(_)) => (nesting.message, Some(nesting.help), Some(nesting.line)), + (Some(nesting), None) => (error.to_owned(), Some(nesting.help), None), + (None, _) => (error.to_owned(), None, None), + } } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/interpolated.rs b/crates/sqlbuild-python-front/src/sqltext/main/interpolated.rs index 37f5d41c82..418249e763 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/interpolated.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/interpolated.rs @@ -1,16 +1,29 @@ //! One SQL text interpolated: the SQL (`None` when unchanged), its reads, and its error. -use crate::sqltext::_helpers::static_sql; use crate::sqltext::_helpers::static_sql::InterpolatedRow; +use crate::sqltext::_helpers::static_sql::PythonInterpolationHost; +use crate::sqltext::_helpers::static_sql::read_rows; use crate::sqltext::models::InterpolationSources; use pyo3::prelude::PyResult; +use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; +use std::cell::RefCell; /// One SQL text interpolated: the SQL (`None` when unchanged), its reads, and its error. -#[inline] pub fn interpolated( sources: InterpolationSources<'_>, sql: &str, file_path: &str, ) -> PyResult { - static_sql::interpolated(sources, sql, file_path) + let host = PythonInterpolationHost { + sources, + raised: RefCell::new(None), + }; + let row = match interpolate_sql(&host, sql, file_path) { + Ok(result) => (result.sql, read_rows(result.reads), None), + Err(failure) => (None, read_rows(failure.reads), Some(failure.message)), + }; + if let Some(error) = host.raised.borrow_mut().take() { + return Err(error); + } + Ok(row) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/interpolation_batch.rs b/crates/sqlbuild-python-front/src/sqltext/main/interpolation_batch.rs index 5c13ea9fdf..3156622ff2 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/interpolation_batch.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/interpolation_batch.rs @@ -1,15 +1,42 @@ //! Each `(sql, file path)` interpolated in order; non-rendering Python failures raise. -use crate::sqltext::_helpers::static_sql; +use crate::sqltext::_helpers::static_sql::PythonInterpolationHost; +use crate::sqltext::_helpers::static_sql::read_rows; use crate::sqltext::models::InterpolationRow; use crate::sqltext::models::InterpolationSources; use pyo3::prelude::PyResult; +use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; +use std::cell::RefCell; /// Each `(sql, file path)` interpolated in order; non-rendering Python failures raise. -#[inline] pub fn interpolation_batch( sources: InterpolationSources<'_>, sqls: &[(String, String)], ) -> PyResult> { - static_sql::interpolation_batch(sources, sqls) + let host = PythonInterpolationHost { + sources, + raised: RefCell::new(None), + }; + let mut rows = Vec::with_capacity(sqls.len()); + for (sql, file_path) in sqls { + let row = match interpolate_sql(&host, sql, file_path) { + Ok(result) => InterpolationRow { + sql: result.sql, + spans: result.spans, + reads: read_rows(result.reads), + error: None, + }, + Err(failure) => InterpolationRow { + sql: None, + spans: Vec::new(), + reads: read_rows(failure.reads), + error: Some(failure.message), + }, + }; + if let Some(error) = host.raised.borrow_mut().take() { + return Err(error); + } + rows.push(row); + } + Ok(rows) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/location_rows.rs b/crates/sqlbuild-python-front/src/sqltext/main/location_rows.rs index 5788549395..bb42cb86fd 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/location_rows.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/location_rows.rs @@ -1,12 +1,19 @@ //! Location rows. -use crate::sqltext::_helpers::model_headers; use crate::sqltext::models::ColumnLocation; /// Location rows. -#[inline] pub fn location_rows( locations: Vec<(String, sqlbuild_sqltext::compiler::models::TextLocation)>, ) -> Vec { - model_headers::location_rows(locations) + locations + .into_iter() + .map(|(name, location)| ColumnLocation { + name, + line: location.line, + column: location.column, + end_line: location.end_line, + end_column: location.end_column, + }) + .collect() } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/map_to_python.rs b/crates/sqlbuild-python-front/src/sqltext/main/map_to_python.rs index eb9e38b163..46add43371 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/map_to_python.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/map_to_python.rs @@ -1,14 +1,15 @@ //! Map to python. -use crate::sqltext::_helpers::authored_values; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::sqltext::main::authored_value_to_python::authored_value_to_python; +use pyo3::prelude::{Py, PyAny, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_sqltext::compiler::models::AuthoredValue; /// Map to python. -#[inline] pub fn map_to_python(py: Python<'_>, values: &[(String, AuthoredValue)]) -> PyResult> { - authored_values::map_to_python(py, values) + let result = PyDict::new(py); + for (key, value) in values { + result.set_item(key, authored_value_to_python(py, value)?)?; + } + Ok(result.unbind().into_any()) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/model_output_columns.rs b/crates/sqlbuild-python-front/src/sqltext/main/model_output_columns.rs index 3c8c84f92b..aa07c51d5b 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/model_output_columns.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/model_output_columns.rs @@ -1,14 +1,23 @@ //! Simple top-level SELECT output names and their authored locations. -use crate::sqltext::_helpers::model_headers; +use crate::sqltext::main::location_rows::location_rows; use crate::sqltext::models::ColumnLocation; use pyo3::prelude::PyResult; +use pyo3::pyfunction; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; /// Simple top-level SELECT output names and their authored locations. -#[inline] +#[pyfunction] pub fn model_output_columns( contents: &str, implicit_aliases: bool, ) -> PyResult> { - model_headers::model_output_columns(contents, implicit_aliases) + compiler_guard(|| { + Ok(location_rows( + sqlbuild_sqltext::compiler::main::model_output_columns::model_output_columns( + contents, + implicit_aliases, + ), + )) + }) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/optional_authored_value_to_python.rs b/crates/sqlbuild-python-front/src/sqltext/main/optional_authored_value_to_python.rs index 6385780a54..01678d05eb 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/optional_authored_value_to_python.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/optional_authored_value_to_python.rs @@ -1,17 +1,15 @@ //! Optional authored value to python. -use crate::sqltext::_helpers::authored_values; -use pyo3::prelude::Py; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use crate::sqltext::main::authored_value_to_python::authored_value_to_python; +use pyo3::prelude::{Py, PyAny, PyResult, Python}; use sqlbuild_sqltext::compiler::models::AuthoredValue; /// Optional authored value to python. -#[inline] pub fn optional_authored_value_to_python( py: Python<'_>, value: Option<&AuthoredValue>, ) -> PyResult>> { - authored_values::optional_authored_value_to_python(py, value) + value + .map(|item| authored_value_to_python(py, item)) + .transpose() } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/optional_record_syntax.rs b/crates/sqlbuild-python-front/src/sqltext/main/optional_record_syntax.rs index d329f2acc3..3ae3df6734 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/optional_record_syntax.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/optional_record_syntax.rs @@ -1,13 +1,13 @@ //! The lexical rules a `SqlLexicalSyntax` record holds, or `None` for `None`. -use crate::sqltext::_helpers::lexical_syntax; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::sqltext::main::record_syntax::record_syntax; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// The lexical rules a `SqlLexicalSyntax` record holds, or `None` for `None`. -#[inline] pub fn optional_record_syntax(value: &Bound<'_, PyAny>) -> PyResult> { - lexical_syntax::optional_record_syntax(value) + if value.is_none() { + return Ok(None); + } + record_syntax(value).map(Some) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs b/crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs index 068fe6c526..cfdf9a884f 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/parse_model_headers.rs @@ -1,15 +1,30 @@ //! Parse model headers. -use crate::sqltext::_helpers::model_headers; +use crate::sqltext::main::optional_authored_value_to_python::optional_authored_value_to_python; use crate::sqltext::types::ParsedModelHeader; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::boundary::main::value_error::value_error; +use sqlbuild_python_base::types::CompilerDetach; -/// Parse model headers. -#[inline] +#[pyfunction] pub fn parse_model_headers( py: Python<'_>, headers: Vec, ) -> PyResult> { - model_headers::parse_model_headers(py, headers) + let parsed = py + .compiler_detach(|| { + sqlbuild_sqltext::compiler::main::model_header_parsing::parse_batch(&headers) + }) + .map_err(value_error)?; + parsed + .into_iter() + .map(|(value, offsets, error)| { + Ok(( + optional_authored_value_to_python(py, value.as_ref())?, + offsets, + error, + )) + }) + .collect() } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/record_syntax.rs b/crates/sqlbuild-python-front/src/sqltext/main/record_syntax.rs index a990fdc113..9ca9b52207 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/record_syntax.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/record_syntax.rs @@ -1,13 +1,20 @@ //! The lexical rules a `SqlLexicalSyntax` record holds. -use crate::sqltext::_helpers::lexical_syntax; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::sqltext::main::sorted_texts::sorted_texts; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// The lexical rules a `SqlLexicalSyntax` record holds. -#[inline] pub fn record_syntax(instance: &Bound<'_, PyAny>) -> PyResult { - lexical_syntax::record_syntax(instance) + let flag = |name: &str| -> PyResult { native_field(instance, name)?.is_truthy() }; + Ok(LexicalSyntax { + backslash_escape_quotes: sorted_texts(&native_field(instance, "backslash_escape_quotes")?)?, + escape_string_prefix: flag("escape_string_prefix")?, + raw_string_prefix: flag("raw_string_prefix")?, + triple_quoted_strings: flag("triple_quoted_strings")?, + nested_block_comments: flag("nested_block_comments")?, + line_comment_prefixes: sorted_texts(&native_field(instance, "line_comment_prefixes")?)?, + bracket_identifiers: false, + }) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/register_code_scanner.rs b/crates/sqlbuild-python-front/src/sqltext/main/register_code_scanner.rs index f70b7bc051..de3c31c66e 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/register_code_scanner.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/register_code_scanner.rs @@ -1,12 +1,10 @@ //! Register the code scanner functions and classes. -use crate::sqltext::_helpers::code_scanner; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::sqltext::models::SqlCodeScanner; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; /// Register the code scanner functions and classes. -#[inline] pub fn register_code_scanner(module: &Bound<'_, PyModule>) -> PyResult<()> { - code_scanner::register(module) + module.add_class::()?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs b/crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs index 6ab4c7d18f..96941c4f62 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs @@ -1,12 +1,15 @@ //! Register the ddl effects functions and classes. -use crate::sqltext::_helpers::ddl_effects; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::sqltext::_helpers::ddl_effects::qualified_relation_name_key; +use crate::sqltext::_helpers::ddl_effects::relation_name_key; +use crate::sqltext::_helpers::ddl_effects::statement_metadata_effect; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the ddl effects functions and classes. -#[inline] pub fn register_ddl_effects(module: &Bound<'_, PyModule>) -> PyResult<()> { - ddl_effects::register(module) + module.add_function(wrap_pyfunction!(statement_metadata_effect, module)?)?; + module.add_function(wrap_pyfunction!(relation_name_key, module)?)?; + module.add_function(wrap_pyfunction!(qualified_relation_name_key, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs b/crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs index abf259e94a..a63bf0c341 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs @@ -1,12 +1,15 @@ //! Register the model headers functions and classes. -use crate::sqltext::_helpers::model_headers; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::sqltext::_helpers::model_headers::model_header_column_locations; +use crate::sqltext::main::model_output_columns::model_output_columns; +use crate::sqltext::main::parse_model_headers::parse_model_headers; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the model headers functions and classes. -#[inline] pub fn register_model_headers(module: &Bound<'_, PyModule>) -> PyResult<()> { - model_headers::register(module) + module.add_function(wrap_pyfunction!(parse_model_headers, module)?)?; + module.add_function(wrap_pyfunction!(model_output_columns, module)?)?; + module.add_function(wrap_pyfunction!(model_header_column_locations, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/register_placeholders.rs b/crates/sqlbuild-python-front/src/sqltext/main/register_placeholders.rs index e49345fc1b..264936f90b 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/register_placeholders.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/register_placeholders.rs @@ -1,12 +1,11 @@ //! Register the placeholders functions and classes. -use crate::sqltext::_helpers::placeholders; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::sqltext::_helpers::placeholders::substitute_placeholder_defaults; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the placeholders functions and classes. -#[inline] pub fn register_placeholders(module: &Bound<'_, PyModule>) -> PyResult<()> { - placeholders::register(module) + module.add_function(wrap_pyfunction!(substitute_placeholder_defaults, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/register_sql_references.rs b/crates/sqlbuild-python-front/src/sqltext/main/register_sql_references.rs index 98649fadff..3c5370803d 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/register_sql_references.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/register_sql_references.rs @@ -1,12 +1,15 @@ //! Register the sql references functions and classes. -use crate::sqltext::_helpers::sql_references; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::sqltext::_helpers::sql_references::SqlReferenceScanner; +use crate::sqltext::_helpers::sql_references::scan_sql_reference_calls; +use crate::sqltext::main::extract_references::extract_references; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the sql references functions and classes. -#[inline] pub fn register_sql_references(module: &Bound<'_, PyModule>) -> PyResult<()> { - sql_references::register(module) + module.add_class::()?; + module.add_function(wrap_pyfunction!(scan_sql_reference_calls, module)?)?; + module.add_function(wrap_pyfunction!(extract_references, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/sorted_texts.rs b/crates/sqlbuild-python-front/src/sqltext/main/sorted_texts.rs index f16064c9b8..7dfb314ad6 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/sorted_texts.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/sorted_texts.rs @@ -1,12 +1,13 @@ //! The sorted texts of a set of strings. -use crate::sqltext::_helpers::lexical_syntax; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// The sorted texts of a set of strings. -#[inline] pub fn sorted_texts(values: &Bound<'_, PyAny>) -> PyResult> { - lexical_syntax::sorted_texts(values) + let mut texts: Vec = values + .try_iter()? + .map(|value| value?.extract::()) + .collect::>()?; + texts.sort_unstable(); + Ok(texts) } diff --git a/crates/sqlbuild-python-front/src/sqltext/main/syntax_record.rs b/crates/sqlbuild-python-front/src/sqltext/main/syntax_record.rs index 6cc6dfdb00..b06b0fd7dd 100644 --- a/crates/sqlbuild-python-front/src/sqltext/main/syntax_record.rs +++ b/crates/sqlbuild-python-front/src/sqltext/main/syntax_record.rs @@ -1,14 +1,32 @@ //! A `SqlLexicalSyntax` record holding `syntax`. -use crate::sqltext::_helpers::lexical_syntax; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; -use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyFrozenSet; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// A `SqlLexicalSyntax` record holding `syntax`. -#[inline] pub fn syntax_record<'py>(py: Python<'py>, syntax: &LexicalSyntax) -> PyResult> { - lexical_syntax::syntax_record(py, syntax) + let flag = |value: bool| -> PyResult> { + Ok(pyo3::types::PyBool::new(py, value).to_owned().into_any()) + }; + new_record_with( + py, + RecordId::SqlLexicalSyntax, + &[ + ( + "backslash_escape_quotes", + PyFrozenSet::new(py, &syntax.backslash_escape_quotes)?.into_any(), + ), + ("escape_string_prefix", flag(syntax.escape_string_prefix)?), + ("raw_string_prefix", flag(syntax.raw_string_prefix)?), + ("triple_quoted_strings", flag(syntax.triple_quoted_strings)?), + ("nested_block_comments", flag(syntax.nested_block_comments)?), + ( + "line_comment_prefixes", + PyFrozenSet::new(py, &syntax.line_comment_prefixes)?.into_any(), + ), + ], + ) } diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs index 58a33d0726..b90aab4107 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs @@ -1,11 +1,11 @@ //! Migration event IDs and the typed literals of one appended event row. +use crate::state_rows::main::event_rows::date_time_typed::date_time_typed; +use crate::state_rows::main::event_rows::presumed_utc::presumed_utc; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::json::errors::JsonEmitError; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_state::events::errors::StateSqlError; @@ -16,8 +16,6 @@ use sqlbuild_state::events::main::migration_event_id::migration_event_id; use sqlbuild_state::events::main::old_name_view_event_id::old_name_view_event_id; use sqlbuild_state::events::models::{EventRelation, StateCell, StateColumnType, StateValue}; -use crate::state_rows::_helpers::event_rows::timestamps::date_time_typed; -use crate::state_rows::_helpers::event_rows::timestamps::presumed_utc; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -37,7 +35,7 @@ pub(crate) fn relation(value: &Bound<'_, PyAny>) -> PyResult { /// The content-addressed ID for one migration performed by one run. #[pyfunction(name = "deterministic_migration_event_id")] #[pyo3(signature = (*, run_id, target_name, origin, destination))] -fn migration_id( +pub(crate) fn migration_id( run_id: &str, target_name: Option<&str>, origin: &Bound<'_, PyAny>, @@ -54,7 +52,7 @@ fn migration_id( /// The content-addressed ID for one column rename performed by one run. #[pyfunction(name = "deterministic_column_migration_event_id")] #[pyo3(signature = (**request))] -fn column_id(request: Option<&Bound<'_, PyDict>>) -> PyResult { +pub(crate) fn column_id(request: Option<&Bound<'_, PyDict>>) -> PyResult { let request = Request::new("deterministic_column_migration_event_id", request)?; let run_id: String = request.get("run_id")?.extract()?; let target_name: Option = request.get("target_name")?.extract()?; @@ -71,7 +69,10 @@ fn column_id(request: Option<&Bound<'_, PyDict>>) -> PyResult { /// The content-addressed ID of one old-name step of one recorded move. #[pyfunction(name = "deterministic_old_name_view_event_id")] #[pyo3(signature = (*, event_type, migration_event_id))] -fn old_name_id(event_type: &Bound<'_, PyAny>, migration_event_id: &str) -> PyResult { +pub(crate) fn old_name_id( + event_type: &Bound<'_, PyAny>, + migration_event_id: &str, +) -> PyResult { let event_type: String = native_field(event_type, "value")?.extract()?; old_name_view_event_id(&event_type, migration_event_id).map_err(emit_error) } @@ -119,10 +120,3 @@ fn rendering_error(py: Python<'_>) -> impl Fn(StateSqlError) -> PyErr + '_ { } } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(migration_id, module)?)?; - module.add_function(wrap_pyfunction!(column_id, module)?)?; - module.add_function(wrap_pyfunction!(old_name_id, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs index e3e00b6dba..17f991f70a 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs @@ -1,41 +1,27 @@ //! Source freshness data versions, their hashes, skip equivalence and age policy. +use crate::state_rows::main::event_rows::in_utc::in_utc; +use crate::state_rows::main::event_rows::offset_aware::offset_aware; +use crate::state_rows::main::event_rows::presumed_utc::presumed_utc; use std::cell::RefCell; use pyo3::exceptions::PyOverflowError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyBool, PyInt, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use sqlbuild_cursor::algebra::models::LocalDateTime; use sqlbuild_state::freshness::errors::FreshnessError; -use sqlbuild_state::freshness::main::age_status::age_status; -use sqlbuild_state::freshness::main::data_version_hash::data_version_hash; -use sqlbuild_state::freshness::main::normalized_data_version::normalized_data_version; -use sqlbuild_state::freshness::main::records_equivalent::records_equivalent; -use sqlbuild_state::freshness::models::{ - AgeStatus, ComparableRecord, DataVersion, FreshnessEventIdentity, -}; +use sqlbuild_state::freshness::models::{ComparableRecord, FreshnessEventIdentity}; use sqlbuild_state::freshness::types::TimestampParser; use crate::cursor::constants::DATETIME_MODULE; use crate::cursor::main::values::local_date_time::local_date_time; -use crate::state_rows::_helpers::event_rows::timestamps::date_time_typed; -use crate::state_rows::_helpers::event_rows::timestamps::in_utc; -use crate::state_rows::_helpers::event_rows::timestamps::offset_aware; -use crate::state_rows::_helpers::event_rows::timestamps::presumed_utc; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::NativeAttr; -use sqlbuild_python_base::records::types::RecordId; -const TIMESTAMP_KIND: &str = "timestamp"; -const INTEGER_KIND: &str = "integer"; -const STRING_KIND: &str = "string"; +pub(crate) const TIMESTAMP_KIND: &str = "timestamp"; +pub(crate) const INTEGER_KIND: &str = "integer"; +pub(crate) const STRING_KIND: &str = "string"; /// Normalize timestamp offsets and precision before computing a persisted observation identity. pub(crate) fn record_event_id(record: &Bound<'_, PyAny>) -> PyResult { @@ -71,14 +57,14 @@ pub(crate) fn record_event_id(record: &Bound<'_, PyAny>) -> PyResult { )) } -fn raised(py: Python<'_>, class: NativeAttr, message: String) -> PyErr { +pub(crate) fn raised(py: Python<'_>, class: NativeAttr, message: String) -> PyErr { match native_attr(py, class).and_then(|class| class.call1((message,))) { Ok(error) => PyErr::from_value(error), Err(error) => error, } } -fn python_error(py: Python<'_>, error: FreshnessError) -> PyErr { +pub(crate) fn python_error(py: Python<'_>, error: FreshnessError) -> PyErr { match error { FreshnessError::Observation(message) => { raised(py, NativeAttr::SourceFreshnessObservationError, message) @@ -90,20 +76,7 @@ fn python_error(py: Python<'_>, error: FreshnessError) -> PyErr { } } -/// Hash the stable source freshness identity used as a graph input. -#[pyfunction(name = "source_freshness_data_version_hash")] -#[pyo3(signature = (*, source_name, strategy, value_kind, data_version))] -pub(crate) fn version_hash( - source_name: &str, - strategy: &str, - value_kind: &str, - data_version: &str, -) -> PyResult { - data_version_hash((source_name, strategy), value_kind, data_version) - .map_err(|error| pyo3::exceptions::PyValueError::new_err(format!("{error:?}"))) -} - -fn observation_error(py: Python<'_>, message: &str) -> PyErr { +pub(crate) fn observation_error(py: Python<'_>, message: &str) -> PyErr { raised( py, NativeAttr::SourceFreshnessObservationError, @@ -111,62 +84,10 @@ fn observation_error(py: Python<'_>, message: &str) -> PyErr { ) } -/// A source freshness value normalized for stable state storage. -#[pyfunction(name = "normalize_source_freshness_data_version")] -#[pyo3(signature = (*, value, value_kind))] -pub(crate) fn normalize<'py>( - value: &Bound<'py, PyAny>, - value_kind: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = value.py(); - if value_kind.eq(TIMESTAMP_KIND)? { - if !date_time_typed(value)? { - return Err(observation_error( - py, - "Timestamp source freshness values must be datetime values", - )); - } - let aware: bool = !native_field(value, "tzinfo")?.is_none(); - let local: LocalDateTime = if aware { - local_date_time(&in_utc(value)?)? - } else { - local_date_time(value)? - }; - let text: String = normalized_data_version(&DataVersion::Timestamp { local, utc: aware }); - return Ok(PyString::new(py, &text).into_any()); - } - if value_kind.eq(INTEGER_KIND)? { - if value.is_instance_of::() || !value.is_instance_of::() { - return Err(observation_error( - py, - "Integer source freshness values must be integer values", - )); - } - return Ok(value.str()?.into_any()); - } - if value_kind.eq(STRING_KIND)? { - if !value.is_instance_of::() { - return Err(observation_error( - py, - "String source freshness values must be string values", - )); - } - return Ok(value.clone()); - } - Err(raised( - py, - NativeAttr::SourceFreshnessObservationError, - format!( - "Unsupported source freshness value kind: {}", - value_kind.str()? - ), - )) -} - /// Stored timestamp text parsed with `datetime.fromisoformat`, remembering a Python failure. -struct StateTimestamps<'py> { - py: Python<'py>, - failure: RefCell>, +pub(crate) struct StateTimestamps<'py> { + pub(crate) py: Python<'py>, + pub(crate) failure: RefCell>, } impl StateTimestamps<'_> { @@ -200,7 +121,7 @@ impl TimestampParser for StateTimestamps<'_> { } } -fn comparable(record: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn comparable(record: &Bound<'_, PyAny>) -> PyResult { Ok(ComparableRecord { data_version_hash: native_field(record, "data_version_hash")?.extract()?, value_kind: native_field(record, "value_kind")?.extract()?, @@ -208,31 +129,8 @@ fn comparable(record: &Bound<'_, PyAny>) -> PyResult { }) } -/// Whether two source freshness records are equivalent for skip decisions. -#[pyfunction(name = "source_freshness_records_equivalent")] -#[pyo3(signature = (*, previous_record, current_record, lag_tolerance = None))] -pub(crate) fn equivalent( - previous_record: &Bound<'_, PyAny>, - current_record: &Bound<'_, PyAny>, - lag_tolerance: Option<&str>, -) -> PyResult { - let py: Python<'_> = previous_record.py(); - let parser = StateTimestamps { - py, - failure: RefCell::new(None), - }; - let records = (&comparable(previous_record)?, &comparable(current_record)?); - records_equivalent(records, lag_tolerance, &parser).map_err(|error| { - parser - .failure - .borrow_mut() - .take() - .unwrap_or_else(|| python_error(py, error)) - }) -} - /// A `datetime` required to be aware, in UTC. -fn aware_utc(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult { +pub(crate) fn aware_utc(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult { if !offset_aware(value)? { return Err(observation_error( value.py(), @@ -241,45 +139,3 @@ fn aware_utc(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult( - policy: &Bound<'py, PyAny>, - data_version: &Bound<'py, PyAny>, - observed_at: &Bound<'py, PyAny>, -) -> PyResult>> { - let py: Python<'py> = policy.py(); - if policy.is_none() { - return Ok(None); - } - let statuses: Bound<'py, PyAny> = record_class(py, RecordId::SourceFreshnessAgeStatus)?; - if !date_time_typed(data_version)? { - return native_field(&statuses, "UNKNOWN").map(Some); - } - let version: LocalDateTime = aware_utc(data_version, "data_version")?; - let observed: LocalDateTime = aware_utc(observed_at, "observed_at")?; - let error_after: Option = native_field(policy, "error_after")?.extract()?; - let warn_after: Option = native_field(policy, "warn_after")?.extract()?; - let status: AgeStatus = age_status( - (&version, &observed), - error_after.as_deref(), - warn_after.as_deref(), - ) - .map_err(|error| python_error(py, error))?; - let member: &str = match status { - AgeStatus::Pass => "PASS", - AgeStatus::Warn => "WARN", - AgeStatus::Error => "ERROR", - }; - enum_member(py, RecordId::SourceFreshnessAgeStatus, member).map(Some) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(version_hash, module)?)?; - module.add_function(wrap_pyfunction!(normalize, module)?)?; - module.add_function(wrap_pyfunction!(equivalent, module)?)?; - module.add_function(wrap_pyfunction!(age_policy, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/mod.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/mod.rs index fd6367a2fd..6dfde51710 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/mod.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/mod.rs @@ -1,3 +1,2 @@ pub(crate) mod events; pub(crate) mod freshness; -pub(crate) mod timestamps; diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/timestamps.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/timestamps.rs deleted file mode 100644 index 645abff126..0000000000 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/timestamps.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! Python `datetime` values normalized to UTC wall time. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_cursor::algebra::models::LocalDateTime; - -use crate::cursor::constants::DATETIME_MODULE; -use crate::cursor::main::values::local_date_time::local_date_time; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -/// Whether `value` is a `datetime`. -pub(crate) fn date_time_typed(value: &Bound<'_, PyAny>) -> PyResult { - value.is_instance(&imported(value.py(), DATETIME_MODULE, "datetime")?) -} - -/// Whether `value` has a `tzinfo` that reports an offset. -pub(crate) fn offset_aware(value: &Bound<'_, PyAny>) -> PyResult { - Ok(!native_field(value, "tzinfo")?.is_none() && !value.call_method0("utcoffset")?.is_none()) -} - -/// `value.astimezone(UTC)`. -pub(crate) fn in_utc<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - value.call_method1( - "astimezone", - (imported(value.py(), DATETIME_MODULE, "UTC")?,), - ) -} - -/// The UTC wall time of `value`, reading a naive or offset-less value as UTC. -pub(crate) fn presumed_utc(value: &Bound<'_, PyAny>) -> PyResult { - if offset_aware(value)? { - return local_date_time(&in_utc(value)?); - } - local_date_time(value) -} diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs index 94b847285c..7d8b673ad1 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs @@ -1,41 +1,28 @@ //! Read and append migration state events through the adapter's execute callable. +use crate::state_rows::main::event_rows::presumed_utc::presumed_utc; +use crate::state_rows::main::migrations::layout::layout; +use crate::state_rows::main::migrations::read_events::read_events; use pyo3::exceptions::PyException; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyString, PyTuple, PyTupleMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; use sqlbuild_state::events::main::recorded_renames::recorded_renames; use sqlbuild_state::events::models::RecordedRenameKey; use crate::state_rows::_helpers::event_rows::events::relation as event_relation; -use crate::state_rows::_helpers::event_rows::timestamps::presumed_utc; -use crate::state_rows::_helpers::migrations::migration_objects::event_order; -use crate::state_rows::_helpers::migrations::migration_objects::layout; -use crate::state_rows::_helpers::migrations::migration_objects::state_error; -use crate::state_rows::_helpers::migrations::migration_projections::old_name_views; -use crate::state_rows::_helpers::migrations::migration_rows::decoded_events; -use crate::state_rows::_helpers::migrations::migration_rows::read_statement; +use crate::state_rows::_helpers::migrations::migration_objects::{event_order, state_error}; use crate::state_rows::_helpers::migrations::migration_rows::write_statements; -use crate::state_rows::constants::COLUMN_KIND; -use crate::state_rows::constants::LOGGING_MODULE; -use crate::state_rows::constants::MIGRATION_LOGGER; -use crate::state_rows::constants::MIGRATION_WRITE_ATTEMPTS; -use crate::state_rows::constants::MIGRATION_WRITE_RETRY_BASE_SECONDS; -use crate::state_rows::constants::MIGRATION_WRITE_RETRY_MESSAGE; -use crate::state_rows::constants::MODEL_KIND; -use crate::state_rows::constants::OLD_NAME_KIND; -use crate::state_rows::constants::TIME_MODULE; -use crate::state_rows::models::EventTable; -use crate::state_rows::models::EventTableLayout; -use crate::state_rows::models::StateAccess; +use crate::state_rows::constants::{ + COLUMN_KIND, LOGGING_MODULE, MIGRATION_LOGGER, MIGRATION_WRITE_ATTEMPTS, + MIGRATION_WRITE_RETRY_BASE_SECONDS, MIGRATION_WRITE_RETRY_MESSAGE, TIME_MODULE, +}; +use crate::state_rows::models::{EventTable, EventTableLayout, StateAccess}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::fields::field::field as native_field; -fn execute<'py>(access: &StateAccess<'py>, sql: &str) -> PyResult> { +pub(crate) fn execute<'py>(access: &StateAccess<'py>, sql: &str) -> PyResult> { let py: Python<'py> = access.execute.py(); call_with( &access.execute, @@ -46,7 +33,11 @@ fn execute<'py>(access: &StateAccess<'py>, sql: &str) -> PyResult, error: PyErr) -> PyErr { +pub(crate) fn unreadable( + layout: &EventTableLayout, + schema: &Bound<'_, PyAny>, + error: PyErr, +) -> PyErr { let py: Python<'_> = schema.py(); let describe = |value: &Bound<'_, PyAny>| -> String { value.str().map(|text| text.to_string()).unwrap_or_default() @@ -62,56 +53,6 @@ fn unreadable(layout: &EventTableLayout, schema: &Bound<'_, PyAny>, error: PyErr wrapped } -/// Every event of `layout`'s table this version reads from one schema, from `stored` columns when known. -pub(crate) fn read_events<'py>( - access: &StateAccess<'py>, - layout: &EventTableLayout, - (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), - stored: Option<&[String]>, -) -> PyResult> { - let py: Python<'py> = schema.py(); - let (sql, columns): (String, Vec) = read_statement( - layout, - &access.render_qualified_name, - (database, schema.clone()), - stored, - )?; - let rows: Bound<'py, PyAny> = - match execute(access, &sql).and_then(|cursor| cursor.call_method0("fetchall")) { - Ok(rows) => rows, - Err(error) if error.is_instance_of::(py) => { - return Err(unreadable(layout, &schema, error)); - } - Err(error) => return Err(error), - }; - decoded_events(layout.table, &rows, &columns) -} - -/// The column names of one schema's old-name view state table, as the adapter lists them. -pub(crate) fn stored_old_name_columns<'py>( - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, - (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), -) -> PyResult> { - let py: Python<'py> = adapter.py(); - let columns: Bound<'py, PyAny> = call_with( - &native_field(adapter, "get_columns")?, - &[ - ("connection", connection.clone()), - ("database", database), - ("schema", schema), - ( - "name", - PyString::new(py, OLD_NAME_VIEW_TABLE_NAME).into_any(), - ), - ], - )?; - columns - .try_iter()? - .map(|column| native_field(&column?, "name")?.extract()) - .collect() -} - fn subject(table: EventTable, event: &Bound<'_, PyAny>) -> PyResult { let text = |value: Bound<'_, PyAny>| -> PyResult { Ok(value.str()?.to_string()) }; Ok(match table { @@ -203,7 +144,7 @@ pub(crate) fn access<'py>(request: &Request<'_, 'py>) -> PyResult>) -> PyResult>> { +pub(crate) fn stored_names(stored: Option>) -> PyResult>> { match stored { Some(stored) if !stored.is_none() => stored .try_iter()? @@ -217,7 +158,9 @@ fn stored_names(stored: Option>) -> PyResult /// Every event of one migration state table this version reads from one schema. #[pyfunction(name = "read_migration_state_events")] #[pyo3(signature = (**request))] -fn read_state_events<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn read_state_events<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("read_migration_state_events", request)?; let stored: Option> = stored_names(request.optional("stored_columns")?)?; read_events( @@ -228,32 +171,12 @@ fn read_state_events<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult( - request: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let request = Request::new("read_old_name_view_histories", request)?; - let access: StateAccess<'py> = access(&request)?; - let location = || -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - Ok((request.get("database")?, request.get("schema")?)) - }; - let stored: Option> = stored_names(Some(request.get("stored_columns")?))?; - let moves: Bound<'py, PyTuple> = read_events(&access, &layout(MODEL_KIND)?, location()?, None)?; - let facts: Bound<'py, PyTuple> = read_events( - &access, - &layout(OLD_NAME_KIND)?, - location()?, - stored.as_deref(), - )?; - old_name_views(moves.as_any(), facts.as_any()) -} - /// Renames recorded on the relation at or after `since`, oldest first. #[pyfunction(name = "read_recorded_column_renames")] #[pyo3(signature = (**request))] -fn read_renames<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn read_renames<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("read_recorded_column_renames", request)?; let py: Python<'py> = request.py(); let relation: Bound<'py, PyAny> = request.get("relation")?; @@ -289,25 +212,10 @@ fn read_renames<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult( - adapter: &Bound<'py, PyAny>, - connection: &Bound<'py, PyAny>, - database: Bound<'py, PyAny>, - schema: Bound<'py, PyAny>, -) -> PyResult> { - PyTuple::new( - adapter.py(), - stored_old_name_columns(adapter, connection, (database, schema))?, - ) -} - /// Create the state table from adapter DDL when given, then append the event unless recorded. #[pyfunction(name = "write_migration_state_event")] #[pyo3(signature = (**request))] -fn write_state_event(request: Option<&Bound<'_, PyDict>>) -> PyResult<()> { +pub(crate) fn write_state_event(request: Option<&Bound<'_, PyDict>>) -> PyResult<()> { let request = Request::new("write_migration_state_event", request)?; let create_table_sql: Option = request.get("create_table_sql")?.extract()?; let attempts: i64 = match request.optional("attempts")? { @@ -324,12 +232,3 @@ fn write_state_event(request: Option<&Bound<'_, PyDict>>) -> PyResult<()> { attempts, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(read_state_events, module)?)?; - module.add_function(wrap_pyfunction!(read_histories, module)?)?; - module.add_function(wrap_pyfunction!(read_renames, module)?)?; - module.add_function(wrap_pyfunction!(stored_columns_binding, module)?)?; - module.add_function(wrap_pyfunction!(write_state_event, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_objects.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_objects.rs index 64fdb804f0..e6616cbe36 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_objects.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_objects.rs @@ -1,63 +1,16 @@ //! Migration state table layouts and the Python objects stored in them. -use pyo3::exceptions::PyValueError; +use crate::state_rows::main::event_rows::presumed_utc::presumed_utc; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyString; -use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; -use sqlbuild_core::constants::MIGRATION_TABLE_NAME; -use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; -use sqlbuild_state::events::constants::{ - COLUMN_MIGRATION_COLUMNS, MIGRATION_COLUMNS, OLD_NAME_VIEW_COLUMNS, -}; use sqlbuild_state::events::models::EventOrder; -use crate::state_rows::_helpers::event_rows::timestamps::presumed_utc; -use crate::state_rows::constants::COLUMN_KIND; -use crate::state_rows::constants::MODEL_KIND; -use crate::state_rows::constants::OLD_NAME_KIND; -use crate::state_rows::models::EventTable; use crate::state_rows::models::EventTableLayout; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; -/// The layout of the state table `kind` names: `model`, `column` or `old_name`. -pub(crate) fn layout(kind: &str) -> PyResult { - match kind { - MODEL_KIND => Ok(EventTableLayout { - table: EventTable::Model, - name: MIGRATION_TABLE_NAME, - columns: &MIGRATION_COLUMNS, - relation_attribute: "destination", - state_message: "model migration state requires a target schema", - event_message: "model migration events require a destination schema", - read_subject: "model migration events", - }), - COLUMN_KIND => Ok(EventTableLayout { - table: EventTable::Column, - name: COLUMN_MIGRATION_TABLE_NAME, - columns: &COLUMN_MIGRATION_COLUMNS, - relation_attribute: "relation", - state_message: "column migration state requires a target schema", - event_message: "column migration events require a relation schema", - read_subject: "column migration events", - }), - OLD_NAME_KIND => Ok(EventTableLayout { - table: EventTable::OldName, - name: OLD_NAME_VIEW_TABLE_NAME, - columns: &OLD_NAME_VIEW_COLUMNS, - relation_attribute: "new", - state_message: "old-name view state requires a target schema", - event_message: "old-name view facts require a destination schema", - read_subject: "old-name view facts", - }), - _ => Err(PyValueError::new_err(format!( - "unknown migration state table kind: {kind}" - ))), - } -} - /// `MigrationStateError(message)`. pub(crate) fn state_error(py: Python<'_>, message: &str) -> PyErr { new_plain_error(py, ErrorId::MigrationState, message) diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs index e2e0dc9087..f9a1c6b56a 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs @@ -1,28 +1,21 @@ //! Current-state projections over recorded migration events. +use crate::state_rows::main::migrations::newest_column::newest_column; +use crate::state_rows::main::migrations::newest_move::newest_move; +use crate::state_rows::main::migrations::old_name_views::old_name_views; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_state::events::main::newest_column_event::newest_column_event; -use sqlbuild_state::events::main::newest_move_event::newest_move_event; -use sqlbuild_state::events::main::old_name_histories::old_name_histories; -use sqlbuild_state::events::models::{ - ColumnEventKey, EventOrder, EventRelation, MoveEventKey, OldNameFactKey, OldNameStep, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyTuple; +use sqlbuild_state::events::models::OldNameStep; -use crate::state_rows::_helpers::event_rows::events::relation as event_relation; -use crate::state_rows::_helpers::migrations::migration_objects::event_order; -use crate::state_rows::constants::OLD_NAME_VIEW_CONFIG_KEY; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; -fn collected<'py>(events: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn collected<'py>(events: &Bound<'py, PyAny>) -> PyResult>> { events.try_iter()?.collect() } -fn chosen<'py>( +pub(crate) fn chosen<'py>( py: Python<'py>, events: &[Bound<'py, PyAny>], index: Option, @@ -32,51 +25,7 @@ fn chosen<'py>( .unwrap_or_else(|| py.None().into_bound(py)) } -/// The latest recorded move naming `relation` as origin or destination, or `None`. -pub(crate) fn newest_move<'py>( - events: &Bound<'py, PyAny>, - relation: &Bound<'py, PyAny>, -) -> PyResult> { - let events: Vec> = collected(events)?; - let keys: Vec = events - .iter() - .map(|event| { - Ok(MoveEventKey { - order: event_order(event)?, - origin: event_relation(&native_field(event, "origin")?)?, - destination: event_relation(&native_field(event, "destination")?)?, - }) - }) - .collect::>()?; - let index: Option = newest_move_event(&keys, &event_relation(relation)?); - Ok(chosen(relation.py(), &events, index)) -} - -/// The latest recorded rename on `relation` naming either column, or `None`. -pub(crate) fn newest_column<'py>( - events: &Bound<'py, PyAny>, - relation: &Bound<'py, PyAny>, - (origin_column, destination_column): (&str, &str), -) -> PyResult> { - let events: Vec> = collected(events)?; - let keys: Vec = events - .iter() - .map(|event| { - Ok(ColumnEventKey { - order: event_order(event)?, - relation: event_relation(&native_field(event, "relation")?)?, - origin_column: native_field(event, "origin_column")?.extract()?, - destination_column: native_field(event, "destination_column")?.extract()?, - }) - }) - .collect::>()?; - let target: EventRelation = event_relation(relation)?; - let index: Option = - newest_column_event(&keys, &target, (origin_column, destination_column)); - Ok(chosen(relation.py(), &events, index)) -} - -fn step(fact: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn step(fact: &Bound<'_, PyAny>) -> PyResult { let value: String = native_field(&native_field(fact, "event_type")?, "value")?.extract()?; match value.as_str() { "required" => Ok(OldNameStep::Required), @@ -89,64 +38,9 @@ fn step(fact: &Bound<'_, PyAny>) -> PyResult { } } -/// One history per recorded move whose old name needs a compatibility view, in move order. -pub(crate) fn old_name_views<'py>( - moves: &Bound<'py, PyAny>, - facts: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = moves.py(); - let moves: Vec> = collected(moves)?; - let facts: Vec> = collected(facts)?; - let move_orders: Vec = moves.iter().map(event_order).collect::>()?; - let fact_keys: Vec = facts - .iter() - .map(|fact| { - Ok(OldNameFactKey { - order: event_order(fact)?, - migration_event_id: native_field(fact, "migration_event_id")?.extract()?, - step: step(fact)?, - }) - }) - .collect::>()?; - let fact_at = |index: Option| chosen(py, &facts, index); - let histories: Vec> = old_name_histories(&move_orders, &fact_keys) - .into_iter() - .map(|indexes| { - new_record_with( - py, - RecordId::OldNameViewHistory, - &[ - ("move", moves[indexes.moved].clone()), - ("required", facts[indexes.required].clone()), - ("archived", fact_at(indexes.archived)), - ("created", fact_at(indexes.created)), - ("dropped", fact_at(indexes.dropped)), - ], - ) - }) - .collect::>()?; - PyTuple::new(py, histories) -} - -/// The model's old-name view retention, the project's when unset, or `None` when disabled. -pub(crate) fn old_name_retention<'py>( - config_values: &Bound<'py, PyAny>, - project_retention: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = config_values.py(); - if !config_values.contains(OLD_NAME_VIEW_CONFIG_KEY)? { - return Ok(project_retention.clone()); - } - let value: Bound<'py, PyAny> = config_values.get_item(OLD_NAME_VIEW_CONFIG_KEY)?; - if value.is(PyBool::new(py, false)) { - return Ok(py.None().into_bound(py)); - } - value.str()?.call_method0("strip") -} - #[pyfunction(name = "newest_migration_event_mentioning")] #[pyo3(signature = (*, events, relation))] -fn newest_move_binding<'py>( +pub(crate) fn newest_move_binding<'py>( events: &Bound<'py, PyAny>, relation: &Bound<'py, PyAny>, ) -> PyResult> { @@ -155,7 +49,7 @@ fn newest_move_binding<'py>( #[pyfunction(name = "newest_column_migration_event_mentioning")] #[pyo3(signature = (*, events, relation, origin_column, destination_column))] -fn newest_column_binding<'py>( +pub(crate) fn newest_column_binding<'py>( events: &Bound<'py, PyAny>, relation: &Bound<'py, PyAny>, origin_column: &str, @@ -166,16 +60,9 @@ fn newest_column_binding<'py>( #[pyfunction(name = "project_old_name_views")] #[pyo3(signature = (*, moves, facts))] -fn old_name_views_binding<'py>( +pub(crate) fn old_name_views_binding<'py>( moves: &Bound<'py, PyAny>, facts: &Bound<'py, PyAny>, ) -> PyResult> { old_name_views(moves, facts) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(newest_move_binding, module)?)?; - module.add_function(wrap_pyfunction!(newest_column_binding, module)?)?; - module.add_function(wrap_pyfunction!(old_name_views_binding, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs index 77326fddcd..76132113ad 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs @@ -1,11 +1,11 @@ //! Create, read and append SQL of migration state tables, and stored rows decoded to events. +use crate::state_rows::main::event_rows::date_time_typed::date_time_typed; +use crate::state_rows::main::migrations::layout::layout; use pyo3::exceptions::{PyKeyError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_state::events::constants::TIMESTAMP_COLUMNS; use sqlbuild_state::events::main::column_aliases_json::column_aliases_json; use sqlbuild_state::events::main::decoded_column_aliases::decoded_column_aliases; @@ -17,13 +17,9 @@ use sqlbuild_state::events::main::readable_columns::readable_columns; use crate::cursor::constants::DATETIME_MODULE; use crate::state_rows::_helpers::event_rows::events::typed_insert_sql; -use crate::state_rows::_helpers::event_rows::timestamps::date_time_typed; -use crate::state_rows::_helpers::migrations::migration_objects::layout; -use crate::state_rows::_helpers::migrations::migration_objects::qualified_table; -use crate::state_rows::_helpers::migrations::migration_objects::state_error; +use crate::state_rows::_helpers::migrations::migration_objects::{qualified_table, state_error}; use crate::state_rows::_helpers::tables::state_tables::state_table_create_sql; -use crate::state_rows::models::EventTable; -use crate::state_rows::models::EventTableLayout; +use crate::state_rows::models::{EventTable, EventTableLayout}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; @@ -35,7 +31,7 @@ use sqlbuild_python_base::records::types::RecordId; /// DDL that creates one migration state table when it is missing. #[pyfunction(name = "render_migration_state_create_sql")] #[pyo3(signature = (**request))] -fn create_sql<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn create_sql<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { let request = Request::new("render_migration_state_create_sql", request)?; let py: Python<'py> = request.py(); let layout: EventTableLayout = layout(&request.get("kind")?.extract::()?)?; @@ -414,8 +410,3 @@ pub(crate) fn decoded_events<'py>( } PyTuple::new(py, events) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(create_sql, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs b/crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs index a5f914e949..31f669810e 100644 --- a/crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs +++ b/crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs @@ -1,11 +1,9 @@ //! Append-only state table DDL and the source freshness insert, rendered through adapter callbacks. use pyo3::exceptions::PyKeyError; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyString, PyStringMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_state::freshness::constants::{ @@ -71,7 +69,7 @@ pub(crate) fn state_table_create_sql<'py>( /// Render create-if-missing DDL for string, integer, and text-timestamp state columns. #[pyfunction(name = "render_state_table_create_sql")] #[pyo3(signature = (**request))] -fn render_create_sql(request: Option<&Bound<'_, PyDict>>) -> PyResult { +pub(crate) fn render_create_sql(request: Option<&Bound<'_, PyDict>>) -> PyResult { let request = Request::new("render_state_table_create_sql", request)?; state_table_create_sql( &request.get("qualified_name")?.extract::()?, @@ -99,7 +97,7 @@ fn adapter_user_error(py: Python<'_>, message: &str) -> PyErr { /// Render DML that appends source freshness records. #[pyfunction] #[pyo3(signature = (*, database, schema, records, render_qualified_name))] -fn render_insert_source_freshness_records_sql<'py>( +pub(crate) fn render_insert_source_freshness_records_sql<'py>( database: &Bound<'py, PyAny>, schema: &Bound<'py, PyAny>, records: &Bound<'py, PyAny>, @@ -143,11 +141,3 @@ fn render_insert_source_freshness_records_sql<'py>( &rows, )) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(render_create_sql, module)?)?; - module.add_function(wrap_pyfunction!( - render_insert_source_freshness_records_sql, - module - )?) -} diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/age_policy.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/age_policy.rs index 994837c021..b751740557 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/age_policy.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/age_policy.rs @@ -1,16 +1,48 @@ //! Optional timestamp age policy for one source freshness observation. -use crate::state_rows::_helpers::event_rows::freshness; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::event_rows::freshness::aware_utc; +use crate::state_rows::_helpers::event_rows::freshness::python_error; +use crate::state_rows::main::event_rows::date_time_typed::date_time_typed; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_cursor::algebra::models::LocalDateTime; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_state::freshness::main::age_status::age_status; +use sqlbuild_state::freshness::models::AgeStatus; /// Optional timestamp age policy for one source freshness observation. -#[inline] +#[pyfunction(name = "evaluate_source_freshness_age_policy")] +#[pyo3(signature = (*, policy, data_version, observed_at))] pub fn age_policy<'py>( policy: &Bound<'py, PyAny>, data_version: &Bound<'py, PyAny>, observed_at: &Bound<'py, PyAny>, ) -> PyResult>> { - freshness::age_policy(policy, data_version, observed_at) + let py: Python<'py> = policy.py(); + if policy.is_none() { + return Ok(None); + } + let statuses: Bound<'py, PyAny> = record_class(py, RecordId::SourceFreshnessAgeStatus)?; + if !date_time_typed(data_version)? { + return native_field(&statuses, "UNKNOWN").map(Some); + } + let version: LocalDateTime = aware_utc(data_version, "data_version")?; + let observed: LocalDateTime = aware_utc(observed_at, "observed_at")?; + let error_after: Option = native_field(policy, "error_after")?.extract()?; + let warn_after: Option = native_field(policy, "warn_after")?.extract()?; + let status: AgeStatus = age_status( + (&version, &observed), + error_after.as_deref(), + warn_after.as_deref(), + ) + .map_err(|error| python_error(py, error))?; + let member: &str = match status { + AgeStatus::Pass => "PASS", + AgeStatus::Warn => "WARN", + AgeStatus::Error => "ERROR", + }; + enum_member(py, RecordId::SourceFreshnessAgeStatus, member).map(Some) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/date_time_typed.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/date_time_typed.rs index 7215ae0fd5..f209a3cddb 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/date_time_typed.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/date_time_typed.rs @@ -1,12 +1,10 @@ //! Whether `value` is a `datetime`. -use crate::state_rows::_helpers::event_rows::timestamps; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::constants::DATETIME_MODULE; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::boundary::main::imported::imported; /// Whether `value` is a `datetime`. -#[inline] pub fn date_time_typed(value: &Bound<'_, PyAny>) -> PyResult { - timestamps::date_time_typed(value) + value.is_instance(&imported(value.py(), DATETIME_MODULE, "datetime")?) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/equivalent.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/equivalent.rs index cb8bd180b5..5c3da9029d 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/equivalent.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/equivalent.rs @@ -1,16 +1,32 @@ //! Whether two source freshness records are equivalent for skip decisions. -use crate::state_rows::_helpers::event_rows::freshness; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::event_rows::freshness::StateTimestamps; +use crate::state_rows::_helpers::event_rows::freshness::comparable; +use crate::state_rows::_helpers::event_rows::freshness::python_error; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_state::freshness::main::records_equivalent::records_equivalent; +use std::cell::RefCell; /// Whether two source freshness records are equivalent for skip decisions. -#[inline] +#[pyfunction(name = "source_freshness_records_equivalent")] +#[pyo3(signature = (*, previous_record, current_record, lag_tolerance = None))] pub fn equivalent( previous_record: &Bound<'_, PyAny>, current_record: &Bound<'_, PyAny>, lag_tolerance: Option<&str>, ) -> PyResult { - freshness::equivalent(previous_record, current_record, lag_tolerance) + let py: Python<'_> = previous_record.py(); + let parser = StateTimestamps { + py, + failure: RefCell::new(None), + }; + let records = (&comparable(previous_record)?, &comparable(current_record)?); + records_equivalent(records, lag_tolerance, &parser).map_err(|error| { + parser + .failure + .borrow_mut() + .take() + .unwrap_or_else(|| python_error(py, error)) + }) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/in_utc.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/in_utc.rs index 8d28e7e853..208245a749 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/in_utc.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/in_utc.rs @@ -1,12 +1,13 @@ //! `value.astimezone(UTC)`. -use crate::state_rows::_helpers::event_rows::timestamps; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::constants::DATETIME_MODULE; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::boundary::main::imported::imported; /// `value.astimezone(UTC)`. -#[inline] pub fn in_utc<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - timestamps::in_utc(value) + value.call_method1( + "astimezone", + (imported(value.py(), DATETIME_MODULE, "UTC")?,), + ) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/normalize.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/normalize.rs index ec853a1552..794807f328 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/normalize.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/normalize.rs @@ -1,15 +1,70 @@ //! A source freshness value normalized for stable state storage. -use crate::state_rows::_helpers::event_rows::freshness; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::main::values::local_date_time::local_date_time; +use crate::state_rows::_helpers::event_rows::freshness::INTEGER_KIND; +use crate::state_rows::_helpers::event_rows::freshness::STRING_KIND; +use crate::state_rows::_helpers::event_rows::freshness::TIMESTAMP_KIND; +use crate::state_rows::_helpers::event_rows::freshness::observation_error; +use crate::state_rows::_helpers::event_rows::freshness::raised; +use crate::state_rows::main::event_rows::date_time_typed::date_time_typed; +use crate::state_rows::main::event_rows::in_utc::in_utc; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyBool, PyInt, PyString}; +use sqlbuild_cursor::algebra::models::LocalDateTime; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::NativeAttr; +use sqlbuild_state::freshness::main::normalized_data_version::normalized_data_version; +use sqlbuild_state::freshness::models::DataVersion; /// A source freshness value normalized for stable state storage. -#[inline] +#[pyfunction(name = "normalize_source_freshness_data_version")] +#[pyo3(signature = (*, value, value_kind))] pub fn normalize<'py>( value: &Bound<'py, PyAny>, value_kind: &Bound<'py, PyAny>, ) -> PyResult> { - freshness::normalize(value, value_kind) + let py: Python<'py> = value.py(); + if value_kind.eq(TIMESTAMP_KIND)? { + if !date_time_typed(value)? { + return Err(observation_error( + py, + "Timestamp source freshness values must be datetime values", + )); + } + let aware: bool = !native_field(value, "tzinfo")?.is_none(); + let local: LocalDateTime = if aware { + local_date_time(&in_utc(value)?)? + } else { + local_date_time(value)? + }; + let text: String = normalized_data_version(&DataVersion::Timestamp { local, utc: aware }); + return Ok(PyString::new(py, &text).into_any()); + } + if value_kind.eq(INTEGER_KIND)? { + if value.is_instance_of::() || !value.is_instance_of::() { + return Err(observation_error( + py, + "Integer source freshness values must be integer values", + )); + } + return Ok(value.str()?.into_any()); + } + if value_kind.eq(STRING_KIND)? { + if !value.is_instance_of::() { + return Err(observation_error( + py, + "String source freshness values must be string values", + )); + } + return Ok(value.clone()); + } + Err(raised( + py, + NativeAttr::SourceFreshnessObservationError, + format!( + "Unsupported source freshness value kind: {}", + value_kind.str()? + ), + )) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs index 88ffd3edd5..e07cb3ac56 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs @@ -1,12 +1,9 @@ //! Whether `value` has a `tzinfo` that reports an offset. -use crate::state_rows::_helpers::event_rows::timestamps; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Whether `value` has a `tzinfo` that reports an offset. -#[inline] pub fn offset_aware(value: &Bound<'_, PyAny>) -> PyResult { - timestamps::offset_aware(value) + Ok(!native_field(value, "tzinfo")?.is_none() && !value.call_method0("utcoffset")?.is_none()) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/presumed_utc.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/presumed_utc.rs index 9d8d99beed..064da5dade 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/presumed_utc.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/presumed_utc.rs @@ -1,13 +1,15 @@ //! The UTC wall time of `value`, reading a naive or offset-less value as UTC. -use crate::state_rows::_helpers::event_rows::timestamps; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::cursor::main::values::local_date_time::local_date_time; +use crate::state_rows::main::event_rows::in_utc::in_utc; +use crate::state_rows::main::event_rows::offset_aware::offset_aware; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_cursor::algebra::models::LocalDateTime; /// The UTC wall time of `value`, reading a naive or offset-less value as UTC. -#[inline] pub fn presumed_utc(value: &Bound<'_, PyAny>) -> PyResult { - timestamps::presumed_utc(value) + if offset_aware(value)? { + return local_date_time(&in_utc(value)?); + } + local_date_time(value) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs index 530de6df36..9c193887ad 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs @@ -1,12 +1,15 @@ //! Register the events functions and classes. -use crate::state_rows::_helpers::event_rows::events; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::event_rows::events::column_id; +use crate::state_rows::_helpers::event_rows::events::migration_id; +use crate::state_rows::_helpers::event_rows::events::old_name_id; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the events functions and classes. -#[inline] pub fn register_events(module: &Bound<'_, PyModule>) -> PyResult<()> { - events::register(module) + module.add_function(wrap_pyfunction!(migration_id, module)?)?; + module.add_function(wrap_pyfunction!(column_id, module)?)?; + module.add_function(wrap_pyfunction!(old_name_id, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs index 61581e44e6..0d8791b56b 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs @@ -1,12 +1,17 @@ //! Register the freshness functions and classes. -use crate::state_rows::_helpers::event_rows::freshness; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::state_rows::main::event_rows::age_policy::age_policy; +use crate::state_rows::main::event_rows::equivalent::equivalent; +use crate::state_rows::main::event_rows::normalize::normalize; +use crate::state_rows::main::event_rows::version_hash::version_hash; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the freshness functions and classes. -#[inline] pub fn register_freshness(module: &Bound<'_, PyModule>) -> PyResult<()> { - freshness::register(module) + module.add_function(wrap_pyfunction!(version_hash, module)?)?; + module.add_function(wrap_pyfunction!(normalize, module)?)?; + module.add_function(wrap_pyfunction!(equivalent, module)?)?; + module.add_function(wrap_pyfunction!(age_policy, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/version_hash.rs b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/version_hash.rs index 544e6287c0..b3a371135c 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/event_rows/version_hash.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/event_rows/version_hash.rs @@ -1,15 +1,18 @@ //! Hash the stable source freshness identity used as a graph input. -use crate::state_rows::_helpers::event_rows::freshness; use pyo3::prelude::PyResult; +use pyo3::pyfunction; +use sqlbuild_state::freshness::main::data_version_hash::data_version_hash; /// Hash the stable source freshness identity used as a graph input. -#[inline] +#[pyfunction(name = "source_freshness_data_version_hash")] +#[pyo3(signature = (*, source_name, strategy, value_kind, data_version))] pub fn version_hash( source_name: &str, strategy: &str, value_kind: &str, data_version: &str, ) -> PyResult { - freshness::version_hash(source_name, strategy, value_kind, data_version) + data_version_hash((source_name, strategy), value_kind, data_version) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(format!("{error:?}"))) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/layout.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/layout.rs index caae386fb9..0a3a1fdc96 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/layout.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/layout.rs @@ -1,11 +1,48 @@ //! The layout of the state table `kind` names: `model`, `column` or `old_name`. -use crate::state_rows::_helpers::migrations::migration_objects; -use crate::state_rows::models::EventTableLayout; +use crate::state_rows::constants::{COLUMN_KIND, MODEL_KIND, OLD_NAME_KIND}; +use crate::state_rows::models::{EventTable, EventTableLayout}; +use pyo3::exceptions::PyValueError; use pyo3::prelude::PyResult; +use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; +use sqlbuild_state::events::constants::{ + COLUMN_MIGRATION_COLUMNS, MIGRATION_COLUMNS, OLD_NAME_VIEW_COLUMNS, +}; /// The layout of the state table `kind` names: `model`, `column` or `old_name`. -#[inline] pub fn layout(kind: &str) -> PyResult { - migration_objects::layout(kind) + match kind { + MODEL_KIND => Ok(EventTableLayout { + table: EventTable::Model, + name: MIGRATION_TABLE_NAME, + columns: &MIGRATION_COLUMNS, + relation_attribute: "destination", + state_message: "model migration state requires a target schema", + event_message: "model migration events require a destination schema", + read_subject: "model migration events", + }), + COLUMN_KIND => Ok(EventTableLayout { + table: EventTable::Column, + name: COLUMN_MIGRATION_TABLE_NAME, + columns: &COLUMN_MIGRATION_COLUMNS, + relation_attribute: "relation", + state_message: "column migration state requires a target schema", + event_message: "column migration events require a relation schema", + read_subject: "column migration events", + }), + OLD_NAME_KIND => Ok(EventTableLayout { + table: EventTable::OldName, + name: OLD_NAME_VIEW_TABLE_NAME, + columns: &OLD_NAME_VIEW_COLUMNS, + relation_attribute: "new", + state_message: "old-name view state requires a target schema", + event_message: "old-name view facts require a destination schema", + read_subject: "old-name view facts", + }), + _ => Err(PyValueError::new_err(format!( + "unknown migration state table kind: {kind}" + ))), + } } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_column.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_column.rs index aa92be6ad4..87bcdc7aa0 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_column.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_column.rs @@ -1,16 +1,34 @@ //! The latest recorded rename on `relation` naming either column, or `None`. -use crate::state_rows::_helpers::migrations::migration_projections; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::event_rows::events::relation as event_relation; +use crate::state_rows::_helpers::migrations::migration_objects::event_order; +use crate::state_rows::_helpers::migrations::migration_projections::chosen; +use crate::state_rows::_helpers::migrations::migration_projections::collected; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_state::events::main::newest_column_event::newest_column_event; +use sqlbuild_state::events::models::{ColumnEventKey, EventRelation}; /// The latest recorded rename on `relation` naming either column, or `None`. -#[inline] pub fn newest_column<'py>( events: &Bound<'py, PyAny>, relation: &Bound<'py, PyAny>, (origin_column, destination_column): (&str, &str), ) -> PyResult> { - migration_projections::newest_column(events, relation, (origin_column, destination_column)) + let events: Vec> = collected(events)?; + let keys: Vec = events + .iter() + .map(|event| { + Ok(ColumnEventKey { + order: event_order(event)?, + relation: event_relation(&native_field(event, "relation")?)?, + origin_column: native_field(event, "origin_column")?.extract()?, + destination_column: native_field(event, "destination_column")?.extract()?, + }) + }) + .collect::>()?; + let target: EventRelation = event_relation(relation)?; + let index: Option = + newest_column_event(&keys, &target, (origin_column, destination_column)); + Ok(chosen(relation.py(), &events, index)) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_move.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_move.rs index c24e453093..b65f4ec24d 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_move.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_move.rs @@ -1,15 +1,30 @@ //! The latest recorded move naming `relation` as origin or destination, or `None`. -use crate::state_rows::_helpers::migrations::migration_projections; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::event_rows::events::relation as event_relation; +use crate::state_rows::_helpers::migrations::migration_objects::event_order; +use crate::state_rows::_helpers::migrations::migration_projections::chosen; +use crate::state_rows::_helpers::migrations::migration_projections::collected; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_state::events::main::newest_move_event::newest_move_event; +use sqlbuild_state::events::models::MoveEventKey; /// The latest recorded move naming `relation` as origin or destination, or `None`. -#[inline] pub fn newest_move<'py>( events: &Bound<'py, PyAny>, relation: &Bound<'py, PyAny>, ) -> PyResult> { - migration_projections::newest_move(events, relation) + let events: Vec> = collected(events)?; + let keys: Vec = events + .iter() + .map(|event| { + Ok(MoveEventKey { + order: event_order(event)?, + origin: event_relation(&native_field(event, "origin")?)?, + destination: event_relation(&native_field(event, "destination")?)?, + }) + }) + .collect::>()?; + let index: Option = newest_move_event(&keys, &event_relation(relation)?); + Ok(chosen(relation.py(), &events, index)) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs index 6396c515f7..0291efb114 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs @@ -1,15 +1,21 @@ //! The model's old-name view retention, the project's when unset, or `None` when disabled. -use crate::state_rows::_helpers::migrations::migration_projections; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::constants::OLD_NAME_VIEW_CONFIG_KEY; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyBool; /// The model's old-name view retention, the project's when unset, or `None` when disabled. -#[inline] pub fn old_name_retention<'py>( config_values: &Bound<'py, PyAny>, project_retention: &Bound<'py, PyAny>, ) -> PyResult> { - migration_projections::old_name_retention(config_values, project_retention) + let py: Python<'py> = config_values.py(); + if !config_values.contains(OLD_NAME_VIEW_CONFIG_KEY)? { + return Ok(project_retention.clone()); + } + let value: Bound<'py, PyAny> = config_values.get_item(OLD_NAME_VIEW_CONFIG_KEY)?; + if value.is(PyBool::new(py, false)) { + return Ok(py.None().into_bound(py)); + } + value.str()?.call_method0("strip") } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_views.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_views.rs index 92d839abeb..c8504209b9 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_views.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_views.rs @@ -1,16 +1,52 @@ //! One history per recorded move whose old name needs a compatibility view, in move order. -use crate::state_rows::_helpers::migrations::migration_projections; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::migrations::migration_objects::event_order; +use crate::state_rows::_helpers::migrations::migration_projections::chosen; +use crate::state_rows::_helpers::migrations::migration_projections::collected; +use crate::state_rows::_helpers::migrations::migration_projections::step; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_state::events::main::old_name_histories::old_name_histories; +use sqlbuild_state::events::models::{EventOrder, OldNameFactKey}; /// One history per recorded move whose old name needs a compatibility view, in move order. -#[inline] pub fn old_name_views<'py>( moves: &Bound<'py, PyAny>, facts: &Bound<'py, PyAny>, ) -> PyResult> { - migration_projections::old_name_views(moves, facts) + let py: Python<'py> = moves.py(); + let moves: Vec> = collected(moves)?; + let facts: Vec> = collected(facts)?; + let move_orders: Vec = moves.iter().map(event_order).collect::>()?; + let fact_keys: Vec = facts + .iter() + .map(|fact| { + Ok(OldNameFactKey { + order: event_order(fact)?, + migration_event_id: native_field(fact, "migration_event_id")?.extract()?, + step: step(fact)?, + }) + }) + .collect::>()?; + let fact_at = |index: Option| chosen(py, &facts, index); + let histories: Vec> = old_name_histories(&move_orders, &fact_keys) + .into_iter() + .map(|indexes| { + new_record_with( + py, + RecordId::OldNameViewHistory, + &[ + ("move", moves[indexes.moved].clone()), + ("required", facts[indexes.required].clone()), + ("archived", fact_at(indexes.archived)), + ("created", fact_at(indexes.created)), + ("dropped", fact_at(indexes.dropped)), + ], + ) + }) + .collect::>()?; + PyTuple::new(py, histories) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs index 44beac2afc..55a96e322e 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs @@ -1,20 +1,34 @@ //! Every event of `layout`'s table this version reads from one schema, from `stored` columns when known. -use crate::state_rows::_helpers::migrations::migration_io; -use crate::state_rows::models::EventTableLayout; -use crate::state_rows::models::StateAccess; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::migrations::migration_io::execute; +use crate::state_rows::_helpers::migrations::migration_io::unreadable; +use crate::state_rows::_helpers::migrations::migration_rows::{decoded_events, read_statement}; +use crate::state_rows::models::{EventTableLayout, StateAccess}; +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; /// Every event of `layout`'s table this version reads from one schema, from `stored` columns when known. -#[inline] pub fn read_events<'py>( access: &StateAccess<'py>, layout: &EventTableLayout, (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), stored: Option<&[String]>, ) -> PyResult> { - migration_io::read_events(access, layout, (database, schema), stored) + let py: Python<'py> = schema.py(); + let (sql, columns): (String, Vec) = read_statement( + layout, + &access.render_qualified_name, + (database, schema.clone()), + stored, + )?; + let rows: Bound<'py, PyAny> = + match execute(access, &sql).and_then(|cursor| cursor.call_method0("fetchall")) { + Ok(rows) => rows, + Err(error) if error.is_instance_of::(py) => { + return Err(unreadable(layout, &schema, error)); + } + Err(error) => return Err(error), + }; + decoded_events(layout.table, &rows, &columns) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_histories.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_histories.rs index 1c946a2d48..b8c80a71d8 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_histories.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/read_histories.rs @@ -1,13 +1,33 @@ //! The old-name view history of every recorded move in a schema that holds both state tables. -use crate::state_rows::_helpers::migrations::migration_io; -use pyo3::prelude::Bound; -use pyo3::prelude::PyResult; -use pyo3::types::PyDict; -use pyo3::types::PyTuple; +use crate::state_rows::_helpers::migrations::migration_io::access; +use crate::state_rows::_helpers::migrations::migration_io::stored_names; +use crate::state_rows::constants::{MODEL_KIND, OLD_NAME_KIND}; +use crate::state_rows::main::migrations::layout::layout; +use crate::state_rows::main::migrations::old_name_views::old_name_views; +use crate::state_rows::main::migrations::read_events::read_events; +use crate::state_rows::models::StateAccess; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_python_base::boundary::models::Request; /// The old-name view history of every recorded move in a schema that holds both state tables. -#[inline] +#[pyfunction(name = "read_old_name_view_histories")] +#[pyo3(signature = (**request))] pub fn read_histories<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { - migration_io::read_histories(request) + let request = Request::new("read_old_name_view_histories", request)?; + let access: StateAccess<'py> = access(&request)?; + let location = || -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + Ok((request.get("database")?, request.get("schema")?)) + }; + let stored: Option> = stored_names(Some(request.get("stored_columns")?))?; + let moves: Bound<'py, PyTuple> = read_events(&access, &layout(MODEL_KIND)?, location()?, None)?; + let facts: Bound<'py, PyTuple> = read_events( + &access, + &layout(OLD_NAME_KIND)?, + location()?, + stored.as_deref(), + )?; + old_name_views(moves.as_any(), facts.as_any()) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs index 90b03f6d18..c3167fcd2d 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs @@ -1,12 +1,19 @@ //! Register the migration io functions and classes. -use crate::state_rows::_helpers::migrations::migration_io; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::migrations::migration_io::read_renames; +use crate::state_rows::_helpers::migrations::migration_io::read_state_events; +use crate::state_rows::_helpers::migrations::migration_io::write_state_event; +use crate::state_rows::main::migrations::read_histories::read_histories; +use crate::state_rows::main::migrations::stored_columns_binding::stored_columns_binding; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the migration io functions and classes. -#[inline] pub fn register_migration_io(module: &Bound<'_, PyModule>) -> PyResult<()> { - migration_io::register(module) + module.add_function(wrap_pyfunction!(read_state_events, module)?)?; + module.add_function(wrap_pyfunction!(read_histories, module)?)?; + module.add_function(wrap_pyfunction!(read_renames, module)?)?; + module.add_function(wrap_pyfunction!(stored_columns_binding, module)?)?; + module.add_function(wrap_pyfunction!(write_state_event, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs index 65e9d95e8d..5f07e4919f 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs @@ -1,12 +1,15 @@ //! Register the migration projections functions and classes. -use crate::state_rows::_helpers::migrations::migration_projections; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::migrations::migration_projections::newest_column_binding; +use crate::state_rows::_helpers::migrations::migration_projections::newest_move_binding; +use crate::state_rows::_helpers::migrations::migration_projections::old_name_views_binding; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the migration projections functions and classes. -#[inline] pub fn register_migration_projections(module: &Bound<'_, PyModule>) -> PyResult<()> { - migration_projections::register(module) + module.add_function(wrap_pyfunction!(newest_move_binding, module)?)?; + module.add_function(wrap_pyfunction!(newest_column_binding, module)?)?; + module.add_function(wrap_pyfunction!(old_name_views_binding, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_rows.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_rows.rs index 8a095b7b1e..dce4494d52 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_rows.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_rows.rs @@ -1,12 +1,11 @@ //! Register the migration rows functions and classes. -use crate::state_rows::_helpers::migrations::migration_rows; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::migrations::migration_rows::create_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the migration rows functions and classes. -#[inline] pub fn register_migration_rows(module: &Bound<'_, PyModule>) -> PyResult<()> { - migration_rows::register(module) + module.add_function(wrap_pyfunction!(create_sql, module)?)?; + Ok(()) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_columns_binding.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_columns_binding.rs index dfd775e2ba..d4c0606771 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_columns_binding.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_columns_binding.rs @@ -1,18 +1,21 @@ //! The column names of one schema's old-name view state table. -use crate::state_rows::_helpers::migrations::migration_io; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use crate::state_rows::main::migrations::stored_old_name_columns::stored_old_name_columns; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyTuple; /// The column names of one schema's old-name view state table. -#[inline] +#[pyfunction(name = "stored_old_name_view_columns")] +#[pyo3(signature = (*, adapter, connection, database, schema))] pub fn stored_columns_binding<'py>( adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, database: Bound<'py, PyAny>, schema: Bound<'py, PyAny>, ) -> PyResult> { - migration_io::stored_columns_binding(adapter, connection, database, schema) + PyTuple::new( + adapter.py(), + stored_old_name_columns(adapter, connection, (database, schema))?, + ) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_old_name_columns.rs b/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_old_name_columns.rs index 6c29967353..cf9cc38b80 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_old_name_columns.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/migrations/stored_old_name_columns.rs @@ -1,16 +1,32 @@ //! The column names of one schema's old-name view state table, as the adapter lists them. -use crate::state_rows::_helpers::migrations::migration_io; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; /// The column names of one schema's old-name view state table, as the adapter lists them. -#[inline] pub fn stored_old_name_columns<'py>( adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, (database, schema): (Bound<'py, PyAny>, Bound<'py, PyAny>), ) -> PyResult> { - migration_io::stored_old_name_columns(adapter, connection, (database, schema)) + let py: Python<'py> = adapter.py(); + let columns: Bound<'py, PyAny> = call_with( + &native_field(adapter, "get_columns")?, + &[ + ("connection", connection.clone()), + ("database", database), + ("schema", schema), + ( + "name", + PyString::new(py, OLD_NAME_VIEW_TABLE_NAME).into_any(), + ), + ], + )?; + columns + .try_iter()? + .map(|column| native_field(&column?, "name")?.extract()) + .collect() } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/tables/register_state_tables.rs b/crates/sqlbuild-python-front/src/state_rows/main/tables/register_state_tables.rs index db283b6c98..374cfdc103 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/tables/register_state_tables.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/tables/register_state_tables.rs @@ -1,12 +1,15 @@ //! Register the state tables functions and classes. -use crate::state_rows::_helpers::tables::state_tables; -use pyo3::prelude::Bound; -use pyo3::prelude::PyModule; -use pyo3::prelude::PyResult; +use crate::state_rows::_helpers::tables::state_tables::render_create_sql; +use crate::state_rows::_helpers::tables::state_tables::render_insert_source_freshness_records_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; /// Register the state tables functions and classes. -#[inline] pub fn register_state_tables(module: &Bound<'_, PyModule>) -> PyResult<()> { - state_tables::register(module) + module.add_function(wrap_pyfunction!(render_create_sql, module)?)?; + module.add_function(wrap_pyfunction!( + render_insert_source_freshness_records_sql, + module + )?) } diff --git a/crates/sqlbuild-python-front/src/state_rows/main/tables/state_table_create_sql.rs b/crates/sqlbuild-python-front/src/state_rows/main/tables/state_table_create_sql.rs index 20b153bb59..3657bb3367 100644 --- a/crates/sqlbuild-python-front/src/state_rows/main/tables/state_table_create_sql.rs +++ b/crates/sqlbuild-python-front/src/state_rows/main/tables/state_table_create_sql.rs @@ -1,9 +1,7 @@ //! Create-if-missing DDL whose column types the adapter renders. use crate::state_rows::_helpers::tables::state_tables; -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; -use pyo3::prelude::PyResult; +use pyo3::prelude::{Bound, PyAny, PyResult}; /// Create-if-missing DDL whose column types the adapter renders. #[inline] diff --git a/crates/sqlbuild-python-front/src/state_rows/models.rs b/crates/sqlbuild-python-front/src/state_rows/models.rs index 542ffd658c..1a2a8e7146 100644 --- a/crates/sqlbuild-python-front/src/state_rows/models.rs +++ b/crates/sqlbuild-python-front/src/state_rows/models.rs @@ -1,7 +1,6 @@ //! Public records of state rows. -use pyo3::prelude::Bound; -use pyo3::prelude::PyAny; +use pyo3::prelude::{Bound, PyAny}; /// Which migration state table a request reads or writes. #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/sqlbuild-python-project/Cargo.toml b/crates/sqlbuild-python-project/Cargo.toml new file mode 100644 index 0000000000..dc526705d0 --- /dev/null +++ b/crates/sqlbuild-python-project/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "sqlbuild-python-project" +description = "The project layer of the sqlbuild._native bindings: the compiled project and its views, graph, references, lineage, fingerprints, semantic shapes, planner scope, selection and strategy, and SQL test planning." +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +chrono.workspace = true +chrono-tz.workspace = true +pyo3.workspace = true +serde_json.workspace = true +sha2.workspace = true +sqlbuild-analysis.workspace = true +sqlbuild-attachments.workspace = true +sqlbuild-cache.workspace = true +sqlbuild-cli.workspace = true +sqlbuild-config.workspace = true +sqlbuild-core.workspace = true +sqlbuild-cursor.workspace = true +sqlbuild-discovery.workspace = true +sqlbuild-host.workspace = true +sqlbuild-model-config.workspace = true +sqlbuild-orchestration.workspace = true +sqlbuild-project.workspace = true +sqlbuild-pysource.workspace = true +sqlbuild-python-base.workspace = true +sqlbuild-python-front.workspace = true +sqlbuild-rules.workspace = true +sqlbuild-sqltext.workspace = true +tokio.workspace = true +uuid.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs new file mode 100644 index 0000000000..39d46a71d0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs @@ -0,0 +1,122 @@ +//! The compile-lifetime binding catalog: known relation shapes around one native project catalog. + +use crate::analysis::constants::NATIVE_DIALECT_ALIASES; +use crate::analysis::models::BindingCatalog; +use crate::analysis::models::CatalogKey; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyDictMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_analysis::semantic_validation::models as validation; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; + +use sqlbuild_analysis::semantic_validation::types::BindingRequest; +use sqlbuild_python_base::types::IntoPositional; + +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_base::types::CompilerDetach; + +pub(crate) const QUOTED_IDENTIFIER_DELIMITER: char = '"'; +pub(crate) const SNOWFLAKE_DIALECT: &str = "snowflake"; +pub(crate) const QUOTED_CASE_PARAMETER: &str = "QUOTED_IDENTIFIERS_IGNORE_CASE"; +pub(crate) const SESSION_PARAMETERS: &str = "session_parameters"; +const GENERIC_DIALECT: &str = "generic"; + +/// The native dialect a request's adapter dialect validates as. +fn native_dialect(dialect: &Bound<'_, PyAny>) -> PyResult { + let name: String = if dialect.is_truthy()? { + dialect.extract()? + } else { + GENERIC_DIALECT.to_owned() + }; + Ok(NATIVE_DIALECT_ALIASES + .iter() + .find(|(alias, _)| *alias == name) + .map_or(name.clone(), |(_, native)| (*native).to_owned())) +} + +/// Binding diagnostics for a batch of complete-schema validation requests, in request order. +#[pyfunction] +#[pyo3(signature = (*, requests))] +pub(crate) fn schema_validations<'py>( + py: Python<'py>, + requests: &Bound<'py, PyAny>, +) -> PyResult> { + let requests: Vec> = requests.try_iter()?.collect::>()?; + let mut created: Vec<(CatalogKey, Py)> = Vec::new(); + let mut groups: Vec<(Py, bool, Vec)> = Vec::new(); + for (index, request) in requests.iter().enumerate() { + let key: CatalogKey = CatalogKey { + dialect: native_dialect(&native_field(request, "dialect")?)?, + quoted_ignore_case: native_field(request, "quoted_identifiers_ignore_case")? + .extract()?, + known_functions: native_field(request, "known_functions")?.extract()?, + known_types: native_field(request, "known_types")?.extract()?, + }; + let explicit: Bound<'py, PyAny> = native_field(request, "catalog")?; + let own: bool = !explicit.is_none(); + let catalog: Py = if own { + explicit.extract()? + } else if let Some((_, catalog)) = created.iter().find(|(known, _)| *known == key) { + catalog.clone_ref(py) + } else { + let catalog: Py = Py::new( + py, + BindingCatalog::create(py, key.clone(), &native_field(request, "schema")?)?, + )?; + created.push((key, catalog.clone_ref(py))); + catalog + }; + match groups.iter_mut().find(|(known, _, _)| known.is(&catalog)) { + Some((_, any_explicit, members)) => { + *any_explicit |= own; + members.push(index); + } + None => groups.push((catalog, own, vec![index])), + } + } + let empty: Bound<'py, PyAny> = new_record(py, RecordId::SqlBindingResult, (), None)?; + let mut results: Vec> = vec![empty; requests.len()]; + for (catalog, any_explicit, members) in groups { + let catalog = catalog.borrow(py); + let batch: Vec> = members + .iter() + .map(|index| requests[*index].clone()) + .collect(); + let prepared: Vec = catalog.prepared(py, &batch)?; + let native = catalog.native.borrow(py); + let inner: &validation::ProjectCatalog = &native.inner; + let dialect: &str = &catalog.dialect; + let rows: Vec> = if any_explicit { + py.compiler_detach(|| inner.binding_results(prepared)) + } else { + py.compiler_detach(|| inner.schema_validation_rows(dialect, prepared)) + } + .map_err(compiler_error)?; + if rows.len() != members.len() { + return Err(compiler_error( + "native SQL schema validation returned an invalid batch response", + )); + } + for (index, diagnostics) in members.into_iter().zip(rows) { + let built: Vec> = diagnostics + .into_iter() + .map(|row| { + new_record( + py, + RecordId::SqlBindingDiagnostic, + row.into_positional(py)?, + None, + ) + }) + .collect::>()?; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("diagnostics", PyTuple::new(py, built)?)?; + results[index] = new_record(py, RecordId::SqlBindingResult, (), Some(&keywords))?; + } + } + PyTuple::new(py, results) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/catalog.rs similarity index 76% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs rename to crates/sqlbuild-python-project/src/analysis/_helpers/catalog.rs index a56da309e0..a7abc19801 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/catalog.rs @@ -1,9 +1,13 @@ //! Python methods of the compile-owned binding catalog and its compact analysis jobs. -use crate::bindings::_helpers::analysis::conversions; -use crate::bindings::_helpers::analysis::normalization_results::normalization_results; -use crate::bindings::models::Positional; -use crate::bindings::models::{CatalogInput, CompactAnalysisJob, ProjectCatalog}; +use crate::analysis::_helpers::conversions::analysis_relations; +use crate::analysis::_helpers::conversions::binding_requests; +use crate::analysis::_helpers::conversions::columns; +use crate::analysis::_helpers::conversions::relations; +use crate::analysis::_helpers::native_classes::CatalogInput; +use crate::analysis::_helpers::normalization_results::normalization_results; +use crate::analysis::models::Positional; +use crate::analysis::models::{CompactAnalysisJob, ProjectCatalog}; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; @@ -23,8 +27,8 @@ impl ProjectCatalog { &self, py: Python<'_>, sql: &str, - #[pyo3(from_py_with = conversions::columns)] columns: validation::Columns, - #[pyo3(from_py_with = conversions::relations)] inputs: Relations, + #[pyo3(from_py_with = columns)] columns: validation::Columns, + #[pyo3(from_py_with = relations)] inputs: Relations, ) -> PyResult)>> { py.compiler_detach(|| self.inner.inferred_schema(sql, columns, inputs)) .map_err(compiler_error) @@ -37,17 +41,11 @@ impl ProjectCatalog { .map_err(PyValueError::new_err) } - fn update_relations( - &mut self, - #[pyo3(from_py_with = conversions::relations)] relations: Relations, - ) { + fn update_relations(&mut self, #[pyo3(from_py_with = relations)] relations: Relations) { self.inner.update_relations(relations); } - fn with_relations( - &self, - #[pyo3(from_py_with = conversions::relations)] relations: Relations, - ) -> Self { + fn with_relations(&self, #[pyo3(from_py_with = relations)] relations: Relations) -> Self { Self { inner: self.inner.with_relations(relations), } @@ -55,7 +53,7 @@ impl ProjectCatalog { fn update_analysis( &mut self, - #[pyo3(from_py_with = conversions::analysis_relations)] relations: HashMap< + #[pyo3(from_py_with = analysis_relations)] relations: HashMap< String, (validation::Columns, validation::Columns), >, @@ -65,14 +63,14 @@ impl ProjectCatalog { fn register_override( &mut self, - #[pyo3(from_py_with = conversions::relations)] relations: Relations, + #[pyo3(from_py_with = relations)] relations: Relations, ) -> usize { self.inner.register_override(relations) } fn validation_payload( &self, - #[pyo3(from_py_with = conversions::binding_requests)] requests: Vec, + #[pyo3(from_py_with = binding_requests)] requests: Vec, ) -> PyResult { self.inner .validation_payload(requests) @@ -91,7 +89,7 @@ impl ProjectCatalog { fn binding_results( &self, py: Python<'_>, - #[pyo3(from_py_with = conversions::binding_requests)] requests: Vec, + #[pyo3(from_py_with = binding_requests)] requests: Vec, ) -> PyResult>>> { let rows: Vec> = py .compiler_detach(|| self.inner.binding_results(requests)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/conversions.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/conversions.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/conversions.rs rename to crates/sqlbuild-python-project/src/analysis/_helpers/conversions.rs diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/mod.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/mod.rs new file mode 100644 index 0000000000..8cf928a522 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod binding_catalog; +pub(crate) mod catalog; +pub(crate) mod conversions; +pub(crate) mod native_classes; +pub(crate) mod normalization; +pub(crate) mod normalization_results; +pub(crate) mod positions; +pub(crate) mod relation_names; +pub(crate) mod schema_request; diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/native_classes.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/native_classes.rs new file mode 100644 index 0000000000..cfa44b0871 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/native_classes.rs @@ -0,0 +1,56 @@ +//! Analysis's Python classes and the mapping requests converted to native models at the boundary. + +use crate::analysis::models::Positional; +use pyo3::FromPyObject; +use sqlbuild_analysis::semantic_validation::models as validation; +use sqlbuild_analysis::semantic_validation::models::Expansion; +use sqlbuild_analysis::semantic_validation::types::Relations; + +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct CatalogInput { + pub(crate) dialect: String, + pub(crate) quoted_ignore_case: bool, + pub(crate) known_functions: Vec, + pub(crate) known_types: Vec, + #[pyo3(from_py_with = crate::analysis::_helpers::conversions::relations)] + pub(crate) relations: Relations, +} + +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct PositionInput { + pub(crate) authored: String, + pub(crate) query: String, + pub(crate) expanded: String, + pub(crate) cleaned: String, + pub(crate) passes: Vec>>, +} + +impl From for validation::CatalogInput { + fn from(input: CatalogInput) -> Self { + Self { + dialect: input.dialect, + quoted_ignore_case: input.quoted_ignore_case, + known_functions: input.known_functions, + known_types: input.known_types, + relations: input.relations, + } + } +} + +impl From for validation::PositionInput { + fn from(input: PositionInput) -> Self { + Self { + authored: input.authored, + query: input.query, + expanded: input.expanded, + cleaned: input.cleaned, + passes: input + .passes + .into_iter() + .map(Positional::unwrap_all) + .collect(), + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/normalization.rs similarity index 56% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization.rs rename to crates/sqlbuild-python-project/src/analysis/_helpers/normalization.rs index 94c65584f8..066785deb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/normalization.rs @@ -1,29 +1,16 @@ //! Normalize SQL for analysis and recover binding diagnostics. -use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Py, PyAny, PyResult, Python}; +use pyo3::pyfunction; -use crate::bindings::_helpers::analysis::normalization_results::normalization_results; -use crate::bindings::models::Positional; +use crate::analysis::_helpers::normalization_results::normalization_results; +use crate::analysis::models::Positional; use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::types::CompilerDetach; #[pyfunction] -pub(crate) fn normalize_analysis_sql( - py: Python<'_>, - request: crate::bindings::models::NormalizationInput, -) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::semantic_validation::main::normalize::normalize_analysis_sql( - request.into(), - ) - }) - .map_err(value_error) -} - -#[pyfunction] -fn normalize_dialect_sql(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { +pub(crate) fn normalize_dialect_sql(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { py.compiler_detach(|| { sqlbuild_analysis::semantic_validation::main::normalize_dialect::normalize_dialect_sql( sql, dialect, @@ -34,7 +21,7 @@ fn normalize_dialect_sql(py: Python<'_>, sql: &str, dialect: &str) -> PyResult, dialect: &str, requests: Vec, @@ -50,7 +37,7 @@ fn normalize_analysis_sqls( } #[pyfunction] -fn binding_diagnostics( +pub(crate) fn binding_diagnostics( py: Python<'_>, sql: &str, dialect: &str, @@ -65,11 +52,3 @@ fn binding_diagnostics( .map(Positional::wrap_all) .map_err(value_error) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(normalize_analysis_sql, module)?)?; - module.add_function(wrap_pyfunction!(normalize_dialect_sql, module)?)?; - module.add_function(wrap_pyfunction!(normalize_analysis_sqls, module)?)?; - module.add_function(wrap_pyfunction!(binding_diagnostics, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization_results.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/normalization_results.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/normalization_results.rs rename to crates/sqlbuild-python-project/src/analysis/_helpers/normalization_results.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/positions.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/positions.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/positions.rs rename to crates/sqlbuild-python-project/src/analysis/_helpers/positions.rs index c3c61fcfea..e92320e42d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/positions.rs +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/positions.rs @@ -1,6 +1,7 @@ //! Python methods of authored binding positions. -use crate::bindings::models::{BindingPositions, PositionInput}; +use crate::analysis::_helpers::native_classes::PositionInput; +use crate::analysis::models::BindingPositions; use pyo3::exceptions::PyValueError; use pyo3::{PyResult, pymethods}; use sqlbuild_analysis::semantic_validation::models as validation; diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/relation_names.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/relation_names.rs new file mode 100644 index 0000000000..42d5c7533f --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/relation_names.rs @@ -0,0 +1,25 @@ +//! Relations named in SQL and temporary tables it creates, for hard-coded name checks. + +use crate::analysis::types::RelationRow; +use pyo3::PyErr; +use sqlbuild_analysis::relation_names::errors::RelationNamesFailure; +use sqlbuild_analysis::relation_names::models::RelationName; +use sqlbuild_core::panics::main::native_failure::native_failure; + +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; + +const RELATION_NAMES_CONTEXT: &str = "native relation name extraction"; + +pub(crate) fn relation_row(relation: RelationName) -> RelationRow { + (relation.name, relation.schema, relation.database) +} + +/// An unknown dialect as the wheel's `ValueError`; anything else as an internal failure. +pub(crate) fn relation_names_error(failure: RelationNamesFailure) -> PyErr { + match failure { + RelationNamesFailure::UnknownDialect(message) => compiler_error(message), + RelationNamesFailure::Internal(reason) => { + compiler_error(native_failure(RELATION_NAMES_CONTEXT, &reason)) + } + } +} diff --git a/crates/sqlbuild-python-project/src/analysis/_helpers/schema_request.rs b/crates/sqlbuild-python-project/src/analysis/_helpers/schema_request.rs new file mode 100644 index 0000000000..b306f205d8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/_helpers/schema_request.rs @@ -0,0 +1,3 @@ +//! Build the stable native schema-validation request of one model query. + +pub(crate) const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python-project/src/analysis/constants.rs b/crates/sqlbuild-python-project/src/analysis/constants.rs new file mode 100644 index 0000000000..6a064e13b7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/constants.rs @@ -0,0 +1,11 @@ +//! Public constants of analysis. + +/// Dialects whose quoted identifiers bind case-sensitively unless the session says otherwise. +pub const CASE_SENSITIVE_BINDING_DIALECTS: [&str; 2] = ["snowflake", "postgresql"]; + +/// Adapter dialect names and the native dialect each validates as. +pub const NATIVE_DIALECT_ALIASES: [(&str, &str); 3] = [ + ("postgres", "postgresql"), + ("motherduck", "duckdb"), + ("sqlserver", "tsql"), +]; diff --git a/crates/sqlbuild-python-project/src/analysis/main/extract_sql_relation_names.rs b/crates/sqlbuild-python-project/src/analysis/main/extract_sql_relation_names.rs new file mode 100644 index 0000000000..9b2e06b121 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/extract_sql_relation_names.rs @@ -0,0 +1,28 @@ +//! `(relations, temporary names)` named in `sql`, or None if unparseable. + +use crate::analysis::_helpers::relation_names::relation_names_error; +use crate::analysis::_helpers::relation_names::relation_row; +use crate::analysis::types::RelationRow; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::relation_names::main::extract_relation_names::extract_relation_names; +use sqlbuild_analysis::relation_names::models::ExtractedRelations; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::types::CompilerDetach; + +/// `(relations, temporary names)` named in `sql`, or None if unparseable. +pub fn extract_sql_relation_names( + py: Python<'_>, + sql: &str, + dialect: Option<&str>, +) -> PyResult, Vec)>> { + let extracted = py + .compiler_detach(|| Ok(extract_relation_names(sql, dialect))) + .map_err(compiler_error)? + .map_err(relation_names_error)?; + Ok(extracted.map( + |ExtractedRelations { + relations, + temporary, + }| { (relations.into_iter().map(relation_row).collect(), temporary) }, + )) +} diff --git a/crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs b/crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs new file mode 100644 index 0000000000..9995efd954 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs @@ -0,0 +1,29 @@ +//! `ignores_quoted_case`: whether the connection explicitly enables case-insensitive quoting. + +use crate::analysis::_helpers::binding_catalog::QUOTED_CASE_PARAMETER; +use crate::analysis::_helpers::binding_catalog::SESSION_PARAMETERS; +use crate::analysis::_helpers::binding_catalog::SNOWFLAKE_DIALECT; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyDictMethods, PyResult}; +use pyo3::types::{PyBool, PyDict}; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; + +/// `ignores_quoted_case`: whether the connection explicitly enables case-insensitive quoting. +pub fn ignores_quoted_case( + connection: &Bound<'_, PyAny>, + dialect: &Bound<'_, PyAny>, +) -> PyResult { + if dialect.is_none() || dialect.extract::()? != SNOWFLAKE_DIALECT { + return Ok(false); + } + let parameters: Bound<'_, PyAny> = mapping_get(connection, SESSION_PARAMETERS)?; + let Ok(parameters) = parameters.downcast::() else { + return Ok(false); + }; + for (key, value) in parameters.iter() { + let upper: Bound<'_, PyAny> = key.str()?.call_method0("upper")?; + if upper.eq(QUOTED_CASE_PARAMETER)? && value.is(PyBool::new(key.py(), true)) { + return Ok(true); + } + } + Ok(false) +} diff --git a/crates/sqlbuild-python-project/src/analysis/main/mod.rs b/crates/sqlbuild-python-project/src/analysis/main/mod.rs new file mode 100644 index 0000000000..bb9d5d1db2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/mod.rs @@ -0,0 +1,7 @@ +pub mod extract_sql_relation_names; +pub mod ignores_quoted_case; +pub mod normalize_analysis_sql; +pub mod register_binding_catalog; +pub mod register_normalization; +pub mod schema_validation_request; +pub mod schema_validations; diff --git a/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs b/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs new file mode 100644 index 0000000000..7aa5f7d17d --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/normalize_analysis_sql.rs @@ -0,0 +1,19 @@ +//! Normalize analysis sql. + +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::boundary::main::value_error::value_error; +use sqlbuild_python_base::types::CompilerDetach; + +#[pyfunction] +pub fn normalize_analysis_sql( + py: Python<'_>, + request: crate::analysis::models::NormalizationInput, +) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::semantic_validation::main::normalize::normalize_analysis_sql( + request.into(), + ) + }) + .map_err(value_error) +} diff --git a/crates/sqlbuild-python-project/src/analysis/main/register_binding_catalog.rs b/crates/sqlbuild-python-project/src/analysis/main/register_binding_catalog.rs new file mode 100644 index 0000000000..71b586b3f3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/register_binding_catalog.rs @@ -0,0 +1,13 @@ +//! Register the binding catalog functions and classes. + +use crate::analysis::_helpers::binding_catalog::schema_validations; +use crate::analysis::models::BindingCatalog; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the binding catalog functions and classes. +pub fn register_binding_catalog(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(schema_validations, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs b/crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs new file mode 100644 index 0000000000..46e2e59955 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs @@ -0,0 +1,17 @@ +//! Register the normalization functions and classes. + +use crate::analysis::_helpers::normalization::binding_diagnostics; +use crate::analysis::_helpers::normalization::normalize_analysis_sqls; +use crate::analysis::_helpers::normalization::normalize_dialect_sql; +use crate::analysis::main::normalize_analysis_sql::normalize_analysis_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the normalization functions and classes. +pub fn register_normalization(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(normalize_analysis_sql, module)?)?; + module.add_function(wrap_pyfunction!(normalize_dialect_sql, module)?)?; + module.add_function(wrap_pyfunction!(normalize_analysis_sqls, module)?)?; + module.add_function(wrap_pyfunction!(binding_diagnostics, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs b/crates/sqlbuild-python-project/src/analysis/main/schema_validation_request.rs similarity index 74% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs rename to crates/sqlbuild-python-project/src/analysis/main/schema_validation_request.rs index dd7787f690..88718e56c7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs +++ b/crates/sqlbuild-python-project/src/analysis/main/schema_validation_request.rs @@ -1,33 +1,19 @@ -//! Build the stable native schema-validation request of one model query. - -use std::collections::HashMap; +//! A `SqlSchemaValidationRequest` over the normalized analysis SQL of `parts.query_sql`. +use crate::analysis::_helpers::schema_request::GENERIC_DIALECT; +use crate::analysis::models::SchemaRequestParts; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyDict; use sqlbuild_analysis::semantic_validation::main::normalize::normalize_analysis_sql; use sqlbuild_analysis::semantic_validation::models::NormalizationInput; - use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; - -const GENERIC_DIALECT: &str = "generic"; - -/// The query, relation schemas and binding context of one schema-validation request. -pub(crate) struct SchemaRequestParts<'py> { - pub(crate) query_sql: String, - pub(crate) placeholders: Bound<'py, PyAny>, - pub(crate) dialect: Bound<'py, PyAny>, - pub(crate) schema: Bound<'py, PyAny>, - pub(crate) known_functions: Bound<'py, PyAny>, - pub(crate) known_types: Bound<'py, PyAny>, - pub(crate) catalog: Bound<'py, PyAny>, - pub(crate) quoted_identifiers_ignore_case: bool, -} +use std::collections::HashMap; /// A `SqlSchemaValidationRequest` over the normalized analysis SQL of `parts.query_sql`. -pub(crate) fn schema_validation_request<'py>( +pub fn schema_validation_request<'py>( py: Python<'py>, parts: SchemaRequestParts<'py>, ) -> PyResult> { diff --git a/crates/sqlbuild-python-project/src/analysis/main/schema_validations.rs b/crates/sqlbuild-python-project/src/analysis/main/schema_validations.rs new file mode 100644 index 0000000000..95873c6a8b --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/main/schema_validations.rs @@ -0,0 +1,14 @@ +//! Binding diagnostics for a batch of complete-schema validation requests, in request order. + +use crate::analysis::_helpers::binding_catalog; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyTuple; + +/// Binding diagnostics for a batch of complete-schema validation requests, in request order. +#[inline] +pub fn schema_validations<'py>( + py: Python<'py>, + requests: &Bound<'py, PyAny>, +) -> PyResult> { + binding_catalog::schema_validations(py, requests) +} diff --git a/crates/sqlbuild-python-project/src/analysis/mod.rs b/crates/sqlbuild-python-project/src/analysis/mod.rs new file mode 100644 index 0000000000..73aacbfb09 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/mod.rs @@ -0,0 +1,7 @@ +//! Analysis classes and requests: the binding catalog, relation names, normalization and schema requests. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs b/crates/sqlbuild-python-project/src/analysis/models.rs similarity index 55% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs rename to crates/sqlbuild-python-project/src/analysis/models.rs index 5595f3c0b0..6eaf134b27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs +++ b/crates/sqlbuild-python-project/src/analysis/models.rs @@ -1,85 +1,78 @@ -//! The compile-lifetime binding catalog: known relation shapes around one native project catalog. +//! Public records of analysis. -use std::sync::atomic::{AtomicBool, Ordering}; - -use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyDictMethods, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyBool, PyDict, PyMapping, PyTuple}; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use crate::analysis::_helpers::binding_catalog::QUOTED_IDENTIFIER_DELIMITER; +use crate::analysis::_helpers::conversions; +use crate::analysis::_helpers::native_classes::CatalogInput; +use crate::analysis::constants::CASE_SENSITIVE_BINDING_DIALECTS; +use pyo3::FromPyObject; +use pyo3::exceptions::PyTypeError; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::Bound; +use pyo3::prelude::IntoPyObject; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyDictMethods; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyMapping; +use pyo3::types::PyTuple; use sqlbuild_analysis::semantic_validation::models as validation; -use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; - -use sqlbuild_analysis::semantic_validation::types::{BindingRequest, Relations}; -use sqlbuild_python_base::types::IntoPositional; - -use crate::bindings::_helpers::analysis::conversions; -use crate::bindings::models::{CatalogInput, ProjectCatalog}; +use sqlbuild_analysis::semantic_validation::types::BindingRequest; +use sqlbuild_analysis::semantic_validation::types::Relations; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; -use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; -use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; - -/// Dialects whose quoted identifiers bind case-sensitively unless the session says otherwise. -pub(crate) const CASE_SENSITIVE_BINDING_DIALECTS: [&str; 2] = ["snowflake", "postgresql"]; -/// Adapter dialect names and the native dialect each validates as. -pub(crate) const NATIVE_DIALECT_ALIASES: [(&str, &str); 3] = [ - ("postgres", "postgresql"), - ("motherduck", "duckdb"), - ("sqlserver", "tsql"), -]; -const QUOTED_IDENTIFIER_DELIMITER: char = '"'; -const SNOWFLAKE_DIALECT: &str = "snowflake"; -const QUOTED_CASE_PARAMETER: &str = "QUOTED_IDENTIFIERS_IGNORE_CASE"; -const SESSION_PARAMETERS: &str = "session_parameters"; -const GENERIC_DIALECT: &str = "generic"; +use sqlbuild_python_base::types::FromPositional; +use sqlbuild_python_base::types::IntoPositional; +use std::collections::HashMap; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; /// One native project catalog and the relation shapes the compile has told it about. #[pyclass(module = "sqlbuild._native")] -pub(crate) struct BindingCatalog { +#[derive(Debug)] +pub struct BindingCatalog { #[pyo3(get)] - pub(crate) schemas: Py, + pub schemas: Py, #[pyo3(get)] - pub(crate) dialect: String, + pub dialect: String, #[pyo3(get)] - pub(crate) quoted_ignore_case: bool, + pub quoted_ignore_case: bool, #[pyo3(get)] - pub(crate) analysis_shapes: Py, + pub analysis_shapes: Py, #[pyo3(get)] - pub(crate) expression_shapes: Py, + pub expression_shapes: Py, #[pyo3(get)] - pub(crate) native: Py, + pub native: Py, /// Whether binding preparation has added relations to `schemas`. - pub(crate) extended: AtomicBool, + pub extended: AtomicBool, } /// A catalog's dialect, case mode, known functions and known types. -#[derive(Clone, PartialEq, Eq)] -pub(crate) struct CatalogKey { - pub(crate) dialect: String, - pub(crate) quoted_ignore_case: bool, - pub(crate) known_functions: Vec, - pub(crate) known_types: Vec, +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CatalogKey { + pub dialect: String, + pub quoted_ignore_case: bool, + pub known_functions: Vec, + pub known_types: Vec, } impl BindingCatalog { /// Whether `schemas` holds exactly the relations it was created with and later given. - pub(crate) fn unextended(&self) -> bool { + pub fn unextended(&self) -> bool { !self.extended.load(Ordering::Relaxed) } /// A catalog for `key` that starts with `relations`. - pub(crate) fn create( - py: Python<'_>, - key: CatalogKey, - relations: &Bound<'_, PyAny>, - ) -> PyResult { + pub fn create(py: Python<'_>, key: CatalogKey, relations: &Bound<'_, PyAny>) -> PyResult { let CatalogKey { dialect, quoted_ignore_case, @@ -110,7 +103,7 @@ impl BindingCatalog { } /// `BindingCatalog.prepare`: add unknown closed relations (overrides while in use), then changed ones. - fn prepared( + pub(crate) fn prepared( &self, py: Python<'_>, requests: &[Bound<'_, PyAny>], @@ -171,7 +164,7 @@ impl BindingCatalog { /// `BindingCatalog(dialect=, quoted_ignore_case=, known_functions=, known_types=, relations=)`. #[new] #[pyo3(signature = (**request))] - fn new(py: Python<'_>, request: Option<&Bound<'_, PyDict>>) -> PyResult { + pub(crate) fn new(py: Python<'_>, request: Option<&Bound<'_, PyDict>>) -> PyResult { let request: Bound<'_, PyDict> = request.cloned().unwrap_or_else(|| PyDict::new(py)); let field = |name: &str| -> PyResult> { request.get_item(name)?.ok_or_else(|| { @@ -191,7 +184,7 @@ impl BindingCatalog { /// Send the native catalog only the analysis shapes that changed. #[pyo3(signature = (*, types, nullability))] - fn prepare_analysis( + pub(crate) fn prepare_analysis( &self, py: Python<'_>, types: &Bound<'_, PyAny>, @@ -221,7 +214,11 @@ impl BindingCatalog { } /// A copy whose relations also include `relations`. - fn with_relations(&self, py: Python<'_>, relations: &Bound<'_, PyAny>) -> PyResult { + pub(crate) fn with_relations( + &self, + py: Python<'_>, + relations: &Bound<'_, PyAny>, + ) -> PyResult { let schemas: Bound<'_, PyDict> = self.schemas.bind(py).copy()?; schemas.update(relations.downcast::()?)?; let native: ProjectCatalog = ProjectCatalog { @@ -244,7 +241,7 @@ impl BindingCatalog { /// The output schema with exact-case names where quoted identifiers bind case-sensitively. #[pyo3(signature = (*, sql, columns, inputs))] - fn inferred_schema<'py>( + pub(crate) fn inferred_schema<'py>( &self, py: Python<'py>, sql: &str, @@ -284,125 +281,93 @@ impl BindingCatalog { } } -/// `ignores_quoted_case`: whether the connection explicitly enables case-insensitive quoting. -pub(crate) fn ignores_quoted_case( - connection: &Bound<'_, PyAny>, - dialect: &Bound<'_, PyAny>, -) -> PyResult { - if dialect.is_none() || dialect.extract::()? != SNOWFLAKE_DIALECT { - return Ok(false); - } - let parameters: Bound<'_, PyAny> = mapping_get(connection, SESSION_PARAMETERS)?; - let Ok(parameters) = parameters.downcast::() else { - return Ok(false); - }; - for (key, value) in parameters.iter() { - let upper: Bound<'_, PyAny> = key.str()?.call_method0("upper")?; - if upper.eq(QUOTED_CASE_PARAMETER)? && value.is(PyBool::new(key.py(), true)) { - return Ok(true); - } +/// The query, relation schemas and binding context of one schema-validation request. +#[derive(Debug)] +pub struct SchemaRequestParts<'py> { + pub query_sql: String, + pub placeholders: Bound<'py, PyAny>, + pub dialect: Bound<'py, PyAny>, + pub schema: Bound<'py, PyAny>, + pub known_functions: Bound<'py, PyAny>, + pub known_types: Bound<'py, PyAny>, + pub catalog: Bound<'py, PyAny>, + pub quoted_identifiers_ignore_case: bool, +} + +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub struct NormalizationInput { + pub sql: String, + pub dialect: String, + pub stubs: HashMap, + pub placeholders: HashMap, +} + +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug)] +pub struct ProjectCatalog { + pub inner: validation::ProjectCatalog, +} + +/// One resolved compact analysis batch that runs without borrowing its project catalog. +#[pyclass(module = "sqlbuild._native", frozen)] +pub struct CompactAnalysisJob { + pub inner: validation::CompactAnalysisJob, +} + +impl std::fmt::Debug for CompactAnalysisJob { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CompactAnalysisJob") + .finish_non_exhaustive() } - Ok(false) } -/// The native dialect a request's adapter dialect validates as. -fn native_dialect(dialect: &Bound<'_, PyAny>) -> PyResult { - let name: String = if dialect.is_truthy()? { - dialect.extract()? - } else { - GENERIC_DIALECT.to_owned() - }; - Ok(NATIVE_DIALECT_ALIASES - .iter() - .find(|(alias, _)| *alias == name) - .map_or(name.clone(), |(_, native)| (*native).to_owned())) +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug)] +pub struct BindingPositions { + pub inner: validation::BindingPositions, } -/// Binding diagnostics for a batch of complete-schema validation requests, in request order. -#[pyfunction] -#[pyo3(signature = (*, requests))] -pub(crate) fn schema_validations<'py>( - py: Python<'py>, - requests: &Bound<'py, PyAny>, -) -> PyResult> { - let requests: Vec> = requests.try_iter()?.collect::>()?; - let mut created: Vec<(CatalogKey, Py)> = Vec::new(); - let mut groups: Vec<(Py, bool, Vec)> = Vec::new(); - for (index, request) in requests.iter().enumerate() { - let key: CatalogKey = CatalogKey { - dialect: native_dialect(&native_field(request, "dialect")?)?, - quoted_ignore_case: native_field(request, "quoted_identifiers_ignore_case")? - .extract()?, - known_functions: native_field(request, "known_functions")?.extract()?, - known_types: native_field(request, "known_types")?.extract()?, - }; - let explicit: Bound<'py, PyAny> = native_field(request, "catalog")?; - let own: bool = !explicit.is_none(); - let catalog: Py = if own { - explicit.extract()? - } else if let Some((_, catalog)) = created.iter().find(|(known, _)| *known == key) { - catalog.clone_ref(py) - } else { - let catalog: Py = Py::new( - py, - BindingCatalog::create(py, key.clone(), &native_field(request, "schema")?)?, - )?; - created.push((key, catalog.clone_ref(py))); - catalog - }; - match groups.iter_mut().find(|(known, _, _)| known.is(&catalog)) { - Some((_, any_explicit, members)) => { - *any_explicit |= own; - members.push(index); - } - None => groups.push((catalog, own, vec![index])), +impl From for validation::NormalizationInput { + fn from(input: NormalizationInput) -> Self { + Self { + sql: input.sql, + dialect: input.dialect, + stubs: input.stubs, + placeholders: input.placeholders, } } - let empty: Bound<'py, PyAny> = new_record(py, RecordId::SqlBindingResult, (), None)?; - let mut results: Vec> = vec![empty; requests.len()]; - for (catalog, any_explicit, members) in groups { - let catalog = catalog.borrow(py); - let batch: Vec> = members - .iter() - .map(|index| requests[*index].clone()) - .collect(); - let prepared: Vec = catalog.prepared(py, &batch)?; - let native = catalog.native.borrow(py); - let inner: &validation::ProjectCatalog = &native.inner; - let dialect: &str = &catalog.dialect; - let rows: Vec> = if any_explicit { - py.compiler_detach(|| inner.binding_results(prepared)) - } else { - py.compiler_detach(|| inner.schema_validation_rows(dialect, prepared)) - } - .map_err(compiler_error)?; - if rows.len() != members.len() { - return Err(compiler_error( - "native SQL schema validation returned an invalid batch response", - )); - } - for (index, diagnostics) in members.into_iter().zip(rows) { - let built: Vec> = diagnostics - .into_iter() - .map(|row| { - new_record( - py, - RecordId::SqlBindingDiagnostic, - row.into_positional(py)?, - None, - ) - }) - .collect::>()?; - let keywords: Bound<'py, PyDict> = PyDict::new(py); - keywords.set_item("diagnostics", PyTuple::new(py, built)?)?; - results[index] = new_record(py, RecordId::SqlBindingResult, (), Some(&keywords))?; - } +} + +/// A record of another crate exchanged with Python as its positional tuple. +#[derive(Clone, Debug)] +pub struct Positional(pub T); + +impl Positional { + /// Each record of `items`, for Python. + pub fn wrap_all(items: Vec) -> Vec { + items.into_iter().map(Self).collect() + } + + /// Each record Python sent in `items`. + pub fn unwrap_all(items: Vec) -> Vec { + items.into_iter().map(|item| item.0).collect() } - PyTuple::new(py, results) } -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(schema_validations, module)?)?; - Ok(()) +impl<'py, T: FromPositional> FromPyObject<'py> for Positional { + fn extract_bound(value: &Bound<'py, PyAny>) -> PyResult { + T::from_positional(value).map(Self) + } +} + +impl<'py, T: IntoPositional> IntoPyObject<'py> for Positional { + type Target = PyTuple; + type Output = Bound<'py, PyTuple>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + self.0.into_positional(py) + } } diff --git a/crates/sqlbuild-python-project/src/analysis/types.rs b/crates/sqlbuild-python-project/src/analysis/types.rs new file mode 100644 index 0000000000..dca7b1ab92 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis/types.rs @@ -0,0 +1,4 @@ +//! Public traits and type aliases of analysis. + +/// `(name, schema, database)`. +pub type RelationRow = (String, Option, Option); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/fixture_columns.rs similarity index 86% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs rename to crates/sqlbuild-python-project/src/analysis_session/_helpers/fixture_columns.rs index 1a6890da21..ae8acc15c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/fixture_columns.rs @@ -2,18 +2,18 @@ use std::collections::HashMap; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::analysis_session::main::projection_facts::projection_facts; use sqlbuild_analysis::assembly::analysis_session::models::ProjectionFacts; use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_core::text::main::python_casefold::python_casefold; -use crate::bindings::_helpers::analysis_session::models::FixtureInferenceCount; -use crate::bindings::_helpers::analysis_session::query_columns::ColumnsRow; -use crate::bindings::_helpers::analysis_session::stage_functions::query_column_rows; +use crate::analysis_session::_helpers::stage_functions::query_column_rows; +use crate::analysis_session::models::FixtureInferenceCount; +use crate::analysis_session::types::ColumnsRow; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; @@ -90,7 +90,7 @@ pub(crate) fn infer_fixture_column_facts<'py>( /// Fixture column inferences computed in this process, optionally clearing the count first. #[pyfunction] #[pyo3(signature = (*, reset = false))] -fn fixture_inference_count(py: Python<'_>, reset: bool) -> PyResult { +pub(crate) fn fixture_inference_count(py: Python<'_>, reset: bool) -> PyResult { let count = inference_count(py)?; let mut count = count.borrow_mut(); if reset { @@ -106,8 +106,3 @@ fn inference_count(py: Python<'_>) -> PyResult> .bind(py) .clone()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(infer_fixture_column_facts, module)?)?; - module.add_function(wrap_pyfunction!(fixture_inference_count, module)?) -} diff --git a/crates/sqlbuild-python-project/src/analysis_session/_helpers/mod.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/mod.rs new file mode 100644 index 0000000000..8094738564 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod fixture_columns; +pub(crate) mod nullability_rules; +pub(crate) mod profile_inputs; +pub(crate) mod query_columns; +pub(crate) mod session; +pub(crate) mod stage_functions; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/nullability_rules.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/nullability_rules.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis_session/nullability_rules.rs rename to crates/sqlbuild-python-project/src/analysis_session/_helpers/nullability_rules.rs diff --git a/crates/sqlbuild-python-project/src/analysis_session/_helpers/profile_inputs.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/profile_inputs.rs new file mode 100644 index 0000000000..3c5b046325 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/profile_inputs.rs @@ -0,0 +1,66 @@ +//! Inference-profile settings crossing into native analysis: rule ids, shapes and the catalog. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; +use sqlbuild_host::entries::models::AdapterHost; + +use crate::analysis::constants::NATIVE_DIALECT_ALIASES; +use crate::analysis::models::{BindingCatalog, CatalogKey}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; + +pub(crate) const GENERIC_DIALECT: &str = "generic"; + +/// The nullability rules the native analysis knows, with their native rule ids. +pub(crate) const KNOWN_RULES: [(AdapterHost, &str); 3] = [ + (AdapterHost::FirstArgNullability, "first_arg"), + ( + AdapterHost::ConditionalResultNullability, + "conditional_result", + ), + (AdapterHost::SafeCastNullability, "safe_cast"), +]; + +/// The adapter's own nullability rules and `InferredNullability`, which native analysis calls. +pub(crate) type AdapterRules = Option<(Py, Py)>; + +/// A `{key: value}` text mapping's items in order. +pub(crate) fn pairs(mapping: &Bound<'_, PyAny>) -> PyResult { + mapping_items(mapping)? + .try_iter()? + .map(|item| item?.extract()) + .collect() +} + +/// Python's `NATIVE_DIALECT_ALIASES.get(dialect, dialect)`. +pub(crate) fn aliased_dialect(_py: Python<'_>, dialect: String) -> PyResult { + Ok(NATIVE_DIALECT_ALIASES + .iter() + .find(|(alias, _)| *alias == dialect) + .map_or(dialect.clone(), |(_, native)| (*native).to_owned())) +} + +/// `profile.binding_catalog or create_binding_catalog(...)` with no functions, types or relations. +pub(crate) fn profile_catalog<'py>( + profile: &Bound<'py, PyAny>, + dialect: &str, +) -> PyResult> { + let catalog: Bound<'py, PyAny> = native_field(profile, "binding_catalog")?; + if catalog.is_truthy()? { + return Ok(catalog); + } + let py: Python<'py> = profile.py(); + let catalog: BindingCatalog = BindingCatalog::create( + py, + CatalogKey { + dialect: dialect.to_owned(), + quoted_ignore_case: native_field(profile, "quoted_identifiers_ignore_case")? + .extract()?, + known_functions: Vec::new(), + known_types: Vec::new(), + }, + PyDict::new(py).as_any(), + )?; + Ok(Bound::new(py, catalog)?.into_any()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/query_columns.rs similarity index 81% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs rename to crates/sqlbuild-python-project/src/analysis_session/_helpers/query_columns.rs index f2761d28d6..2ed7350994 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/query_columns.rs @@ -1,9 +1,10 @@ //! Python's per-query column analysis outside the session, run natively. -use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyRef, PyResult, Python}; +use crate::analysis_session::main::query_columns_error::query_columns_error; +use crate::analysis_session::types::ColumnsRow; +use pyo3::prelude::{Py, PyAny, PyRef, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::PyDict; -use pyo3::{PyErr, pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::assembly::analysis_session::errors::QueryColumnsFailure; use sqlbuild_analysis::assembly::analysis_session::main::projection_facts::projection_facts; use sqlbuild_analysis::assembly::analysis_session::main::query_columns::query_columns; use sqlbuild_analysis::assembly::analysis_session::models::{ @@ -14,14 +15,14 @@ use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_python_base::positional_record; -use crate::bindings::_helpers::analysis_session::nullability_rules::{ +use crate::analysis::models::ProjectCatalog; +use crate::analysis_session::_helpers::nullability_rules::{ RuleFailure, nullability_callback, raised, }; -use crate::bindings::models::ProjectCatalog; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::types::CompilerDetach; -const QUERY_COLUMNS_CONTEXT: &str = "native query column analysis"; +pub(crate) const QUERY_COLUMNS_CONTEXT: &str = "native query column analysis"; const PROJECTION_FACTS_CONTEXT: &str = "native fixture projection facts"; positional_record! { @@ -52,8 +53,6 @@ positional_record! { } /// `([(name, type)], NULL literal names, quoted names)`. type ProjectionFactsRow = (Vec<(String, String)>, Vec, Vec); -/// `(succeeded, [(name, type, nullability)] or None, has star)`. -pub(crate) type ColumnsRow = (bool, Option, String)>>, bool); /// Each query's columns, as Python's analysis outside the session inferred them. #[pyfunction] @@ -104,16 +103,6 @@ pub(crate) fn infer_query_columns( .collect()) } -/// A normalization rejection as Python's `ValueError`; anything else as an internal failure. -pub(crate) fn query_columns_error(failure: QueryColumnsFailure) -> PyErr { - match failure { - QueryColumnsFailure::Normalization(message) => compiler_error(message), - QueryColumnsFailure::Internal(reason) => { - compiler_error(native_failure(QUERY_COLUMNS_CONTEXT, &reason)) - } - } -} - fn query_input(row: QueryRow) -> Result { let QueryRow { sql, @@ -152,7 +141,7 @@ fn columns_row(columns: QueryColumns) -> ColumnsRow { /// Python's `_simple_projection_facts`: `(types, NULL literal names, quoted names)`, casefolded. #[pyfunction] -fn fixture_projection_facts( +pub(crate) fn fixture_projection_facts( py: Python<'_>, sql: &str, dialect: Option<&str>, @@ -163,11 +152,6 @@ fn fixture_projection_facts( Ok((facts.types, facts.null_literal_names, facts.quoted_names)) } -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(fixture_projection_facts, module)?)?; - Ok(()) -} - fn column_row(column: ColumnFact) -> (String, Option, String) { (column.name, column.data_type, column.nullability) } diff --git a/crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs new file mode 100644 index 0000000000..bfe801e6b7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs @@ -0,0 +1,304 @@ +//! The native model analysis session and expression-source shapes. + +use crate::analysis_session::models::FamilyRow; +use crate::analysis_session::models::ModelRow; +use crate::analysis_session::models::ProofRow; +use crate::analysis_session::models::RequestRow; +use pyo3::prelude::{Py, PyAny, PyErr, PyRef, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_analysis::assembly::analysis_session::main::expression_shapes::expression_shapes; +use sqlbuild_analysis::assembly::analysis_session::main::model_reference_facts::model_reference_facts; +use sqlbuild_analysis::assembly::analysis_session::main::names_set_operation::names_set_operation; +use sqlbuild_analysis::assembly::analysis_session::models::{ + ColumnFact, DynamicFamily, ExpressionShape, ExpressionShapeRequest, LineageFacts, LineageRow, + ModelOutcome, ModelRequest, PivotModel, PivotOutcome, SessionRequest, SessionStep, +}; +use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; +use sqlbuild_python_base::positional_record; +use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; + +use sqlbuild_core::constants::PANIC_MESSAGE; +use sqlbuild_core::panics::main::is_compiler_panic::is_compiler_panic; +use sqlbuild_core::panics::main::native_failure::native_failure; + +use crate::analysis::models::Positional; +use crate::analysis::models::ProjectCatalog; +use crate::analysis_session::_helpers::nullability_rules::{ + RuleFailure, nullability_callback, raised, +}; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::types::CompilerDetach; + +/// `(name, type, nullability)`. +pub(crate) type ColumnRow = (String, Option, String); +positional_record! { + into; + /// One lineage column: output column, transform and confidence codes and its sources. + pub(crate) struct LineageItem { + output_column: String, + transform_code: u8, + confidence_code: u8, + /// `(resource type, resource, column)` of each source. + sources: Vec<(String, String, String)>, + } +} + +/// `(publications, failures)`. +pub(crate) type StepRow = (Shapes, Vec); + +positional_record! { + into; + /// One model's analysis, lineage source and cleaned SQL. + pub(crate) struct OutcomeRow { + pub(crate) analysis_succeeded: bool, + pub(crate) columns: Option>, + pub(crate) lineage_source: &'static str, + pub(crate) lineage: Vec, + pub(crate) has_star: bool, + pub(crate) star_resolved: bool, + pub(crate) binding_diagnostics: Vec>, + pub(crate) binding_validated: bool, + pub(crate) cleaned_sql: String, + } +} + +/// One model's pivot SQL and declared families. +pub(crate) type PivotModelRow = (String, Vec); + +/// `("absent" | "deferred" | "proof", proof)`. +pub(crate) type ContractRow = (&'static str, Option); + +positional_record! { + extract; + /// The settings and expressions one expression-source shape request infers. + pub(crate) struct ExpressionShapesRow { + pub(crate) dialect: String, + pub(crate) case_sensitive_shapes: bool, + pub(crate) function_return_types: Pairs, + pub(crate) nullability_rules: Option, + pub(crate) expressions: Vec, + } +} +/// `(catalog schema additions, analysis-shape names, whether each model's analysis succeeded)`. +pub(crate) type FinishRow = (Shapes, Vec, Vec); + +/// What an internal native failure of the session names. +const SESSION_CONTEXT: &str = "native model analysis"; +const SHAPES_CONTEXT: &str = "native expression-source shapes"; +/// The native store kind holding finished model analyses. +pub(crate) const MODEL_ANALYSIS_STORE_KIND: &str = "model-analyses"; + +/// An internal native failure of the session as `NativeCompilerError`. +pub(crate) fn internal_error(reason: &str) -> PyErr { + if is_compiler_panic(reason) && reason != PANIC_MESSAGE { + return compiler_error(reason); + } + compiler_error(native_failure(SESSION_CONTEXT, reason)) +} + +/// Each expression's shape, `None` where it has none; adapter rules run as in the session. +#[pyfunction] +#[pyo3(signature = (catalog, request, adapter_rules=None))] +pub(crate) fn infer_expression_source_shapes( + py: Python<'_>, + catalog: PyRef<'_, ProjectCatalog>, + request: ExpressionShapesRow, + adapter_rules: Option<(Py, Py)>, +) -> PyResult>> { + let ExpressionShapesRow { + dialect, + case_sensitive_shapes, + function_return_types, + nullability_rules, + expressions, + } = request; + let rule_failure: RuleFailure = RuleFailure::default(); + let request = ExpressionShapeRequest { + dialect, + case_sensitive_shapes, + function_return_types, + nullability_rules, + nullability_callback: adapter_rules.map(|(rules, nullability)| { + nullability_callback(rules, nullability, rule_failure.clone()) + }), + expressions, + }; + let catalog = &catalog.inner; + let shapes = py.compiler_detach(|| expression_shapes(catalog, &request)); + if let Some(error) = raised(&rule_failure) { + return Err(error); + } + Ok(shapes + .map_err(|reason| compiler_error(native_failure(SHAPES_CONTEXT, &reason)))? + .into_iter() + .map(|shape| match shape { + ExpressionShape::Inferred(pairs) => Some(pairs), + ExpressionShape::Absent => None, + }) + .collect()) +} + +pub(crate) fn session_request(request: RequestRow) -> SessionRequest { + let RequestRow { + dialect, + case_sensitive_shapes, + quoted_ignore_case, + function_return_types, + known_functions, + nullability_rules, + rich_type_inference, + column_types, + column_nullability, + complete_schemas, + catalog_schemas, + dynamic_families_by_table, + models, + } = request; + SessionRequest { + dialect, + case_sensitive_shapes, + quoted_ignore_case, + function_return_types, + known_functions, + nullability_rules, + nullability_callback: None, + rich_type_inference, + column_types, + column_nullability, + complete_schemas, + catalog_schemas, + dynamic_families_by_table: families_by_table(dynamic_families_by_table), + models: models.into_iter().map(model_request).collect(), + forced_order: None, + retain_relations: sqlbuild_python_base::compile_context::main::delta::capturing::capturing( + ), + } +} + +fn model_request(row: ModelRow) -> ModelRequest { + let ModelRow { + name, + query_sql, + placeholders, + reference_rows, + recover_cte_facts, + snapshot_columns, + dynamic_families, + sql_macro_checks, + } = row; + let has_set_operation = names_set_operation(&query_sql); + let facts = model_reference_facts(&reference_rows); + let pivot_sql = if placeholders.is_empty() { + query_sql.clone() + } else { + placeholder_defaults(&query_sql, &placeholders) + }; + ModelRequest { + name, + query_sql, + placeholders, + references: facts.references, + lineage_references: facts.lineage_references, + required_names: facts.required_names, + recover_cte_facts, + has_set_operation, + snapshot_columns, + pivot_sql, + dynamic_families: dynamic_families.into_iter().map(dynamic_family).collect(), + sql_macro_checks, + } +} + +pub(crate) fn step_row(step: SessionStep) -> StepRow { + (step.publications, step.failures) +} + +fn column_row(column: ColumnFact) -> ColumnRow { + (column.name, column.data_type, column.nullability) +} + +fn lineage_items(rows: Vec) -> Vec { + rows.into_iter() + .map(|row| LineageItem { + output_column: row.output_column, + transform_code: row.transform_code, + confidence_code: row.confidence_code, + sources: row.sources, + }) + .collect() +} + +pub(crate) fn families_by_table( + rows: Vec<(String, Vec)>, +) -> Vec<(String, Vec)> { + rows.into_iter() + .map(|(name, families)| (name, families.into_iter().map(dynamic_family).collect())) + .collect() +} + +fn dynamic_family(row: FamilyRow) -> DynamicFamily { + let FamilyRow { + name, + pivot_column, + value_column, + aggregate, + data_type, + name_pattern, + } = row; + DynamicFamily { + name, + pivot_column, + value_column, + aggregate, + data_type, + name_pattern, + } +} + +pub(crate) fn contract_row(outcome: PivotOutcome) -> ContractRow { + match outcome { + PivotOutcome::Absent => ("absent", None), + PivotOutcome::Proof(proof) => ( + "proof", + Some(ProofRow { + output_proven: proof.output_proven, + fixed_columns: proof.fixed_columns.into_iter().map(column_row).collect(), + families: proof.families, + input_relations: proof.input_relations, + failure_reason: proof.failure_reason, + bare_dynamic_pivot: proof.bare_dynamic_pivot, + }), + ), + } +} + +pub(crate) fn outcome_row(outcome: ModelOutcome, cleaned_sql: String) -> OutcomeRow { + let analysis = outcome.analysis; + let (source, lineage) = match analysis.lineage { + LineageFacts::Native(rows) => ("native", lineage_items(rows)), + LineageFacts::NativeFacts(rows) => ("facts", lineage_items(rows)), + }; + OutcomeRow { + analysis_succeeded: analysis.analysis_succeeded, + columns: analysis + .columns + .map(|columns| columns.into_iter().map(column_row).collect()), + lineage_source: source, + lineage, + has_star: analysis.has_star, + star_resolved: analysis.star_resolved, + binding_diagnostics: Positional::wrap_all(analysis.binding_diagnostics), + binding_validated: analysis.binding_validated, + cleaned_sql, + } +} + +pub(crate) fn pivot_model(row: PivotModelRow) -> PivotModel { + let (sql, families) = row; + PivotModel { + sql, + families: families.into_iter().map(dynamic_family).collect(), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs similarity index 72% rename from crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs rename to crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs index 68bc94ec1e..927e17c6b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs @@ -1,34 +1,34 @@ //! Query columns, expression-source shapes and dynamic pivot proofs read from Python objects. +use crate::analysis_session::main::case_sensitive_shapes::case_sensitive_shapes; +use crate::analysis_session::main::dialect_or_generic::dialect_or_generic; +use crate::analysis_session::main::nullability_rules::nullability_rules; +use crate::analysis_session::main::profile_dialect::profile_dialect; +use crate::analysis_session::main::prove_dynamic_column_contracts::prove_dynamic_column_contracts; +use crate::analysis_session::main::return_types::return_types; +use crate::analysis_session::main::shape_rows::shape_rows; use pyo3::IntoPyObjectExt; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; -use crate::bindings::_helpers::analysis_session::profile_inputs::{ - aliased_dialect, case_sensitive_shapes, dialect_or_generic, nullability_rules, profile_catalog, - profile_dialect, return_types, shape_rows, -}; -use crate::bindings::_helpers::analysis_session::query_columns::{ - ColumnsRow, QueryRequestRow, QueryRow, infer_query_columns, +use crate::analysis::models::ProjectCatalog; +use crate::analysis_session::_helpers::profile_inputs::{aliased_dialect, profile_catalog}; +use crate::analysis_session::_helpers::query_columns::{ + QueryRequestRow, QueryRow, infer_query_columns, }; -use crate::bindings::_helpers::analysis_session::session::{ - ContractRow, ExpressionShapesRow, FamilyRow, NativeModelAnalysisSession, PivotModelRow, - PivotRequestRow, infer_expression_source_shapes, prove_dynamic_column_contracts, +use crate::analysis_session::_helpers::session::{ + ContractRow, ExpressionShapesRow, PivotModelRow, infer_expression_source_shapes, }; -use crate::bindings::models::ProjectCatalog; +use crate::analysis_session::models::{FamilyRow, NativeModelAnalysisSession, PivotRequestRow}; +use crate::analysis_session::types::ColumnsRow; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::compile_driver::constants::EXPRESSION_SHAPE_BATCHES; use sqlbuild_python_base::compile_driver::constants::EXPRESSION_SHAPES_INFERRED; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; -use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; -use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::headers::family_specs::family_specs; const BATCH_MODE: &str = "batch"; @@ -45,7 +45,7 @@ const LINEAGE_RESOURCE_TYPES: [(&str, &str); 4] = [ /// Each query's `(succeeded, column rows, has star)`; a native internal failure raises. #[pyfunction] #[pyo3(signature = (**request))] -fn infer_native_query_column_rows<'py>( +pub(crate) fn infer_native_query_column_rows<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("infer_native_query_column_rows", request)?; @@ -59,71 +59,6 @@ fn infer_native_query_column_rows<'py>( ) } -/// Each query's columns as `NativeQueryColumns`; a native internal failure raises. -#[pyfunction] -#[pyo3(signature = (**request))] -pub(crate) fn infer_native_query_columns<'py>( - request: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let request = Request::new("infer_native_query_columns", request)?; - let py: Python<'py> = request.py(); - let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); - let types: Bound<'py, PyAny> = or_empty(request.optional("column_types_by_table")?, &empty)?; - let nullability: Bound<'py, PyAny> = - or_empty(request.optional("column_nullability_by_table")?, &empty)?; - let rows: Vec = query_column_rows( - &request.get("queries")?, - &request.get("profile")?, - (&types, &nullability), - )?; - let mut analyses: Vec> = Vec::with_capacity(rows.len()); - for (succeeded, columns, has_star) in rows { - let columns: Bound<'py, PyAny> = match columns { - None => py.None().into_bound(py), - Some(columns) => { - let mut built: Vec> = Vec::with_capacity(columns.len()); - for (name, data_type, nullability) in columns { - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("name", name)?; - arguments.set_item("type", data_type)?; - arguments.set_item( - "nullability", - enum_from_value(py, RecordId::InferredNullability, nullability)?, - )?; - built.push(new_record( - py, - RecordId::InferredColumn, - (), - Some(&arguments), - )?); - } - PyTuple::new(py, built)?.into_any() - } - }; - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("succeeded", succeeded)?; - arguments.set_item("columns", columns)?; - arguments.set_item("has_star", has_star)?; - analyses.push(new_record( - py, - RecordId::NativeQueryColumns, - (), - Some(&arguments), - )?); - } - PyTuple::new(py, analyses) -} - -/// One shape per expression, cached on the profile's binding catalog; a failure raises. -#[pyfunction] -#[pyo3(signature = (**request))] -pub(crate) fn infer_native_expression_source_shapes<'py>( - request: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let request = Request::new("infer_native_expression_source_shapes", request)?; - expression_source_shapes(&request.get("expressions")?, &request.get("profile")?) -} - /// One shape per expression, cached on `profile`'s binding catalog; a failure raises. pub(crate) fn expression_source_shapes<'py>( expressions: &Bound<'py, PyAny>, @@ -190,7 +125,7 @@ pub(crate) fn expression_source_shapes<'py>( /// The proof row per `(sql, families)` model from the session or `tables`, else None. #[pyfunction] #[pyo3(signature = (**request))] -fn prove_native_dynamic_contracts<'py>( +pub(crate) fn prove_native_dynamic_contracts<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("prove_native_dynamic_contracts", request)?; @@ -259,7 +194,7 @@ pub(crate) fn family_rows(families: &Bound<'_, PyAny>) -> PyResult( +pub(crate) fn or_empty<'py>( value: Option>, empty: &Bound<'py, PyAny>, ) -> PyResult> { @@ -378,14 +313,3 @@ impl OrderedDict for Pairs { Ok(dict.into_any()) } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(infer_native_query_column_rows, module)?)?; - module.add_function(wrap_pyfunction!(infer_native_query_columns, module)?)?; - module.add_function(wrap_pyfunction!( - infer_native_expression_source_shapes, - module - )?)?; - module.add_function(wrap_pyfunction!(prove_native_dynamic_contracts, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/analysis_session/constants.rs b/crates/sqlbuild-python-project/src/analysis_session/constants.rs new file mode 100644 index 0000000000..5ba9dbf98f --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/constants.rs @@ -0,0 +1,3 @@ +//! Public constants of analysis session. + +pub const ADAPTER_RULE_ID: &str = "python"; diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/case_sensitive_shapes.rs b/crates/sqlbuild-python-project/src/analysis_session/main/case_sensitive_shapes.rs new file mode 100644 index 0000000000..ae616cee72 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/case_sensitive_shapes.rs @@ -0,0 +1,20 @@ +//! Python's `inferred_binding_shape` test for keeping authored identifier quoting. + +use crate::analysis::constants::CASE_SENSITIVE_BINDING_DIALECTS; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Python's `inferred_binding_shape` test for keeping authored identifier quoting. +pub fn case_sensitive_shapes( + profile: &Bound<'_, PyAny>, + dialect: &Bound<'_, PyAny>, +) -> PyResult { + if native_field(profile, "quoted_identifiers_ignore_case")?.is_truthy()? { + return Ok(false); + } + if dialect.is_none() { + return Ok(false); + } + let dialect: String = dialect.extract()?; + Ok(CASE_SENSITIVE_BINDING_DIALECTS.contains(&dialect.as_str())) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/dialect_or_generic.rs b/crates/sqlbuild-python-project/src/analysis_session/main/dialect_or_generic.rs new file mode 100644 index 0000000000..a9cd9ce776 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/dialect_or_generic.rs @@ -0,0 +1,12 @@ +//! `dialect or "generic"`. + +use crate::analysis_session::_helpers::profile_inputs::GENERIC_DIALECT; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +/// `dialect or "generic"`. +pub fn dialect_or_generic(dialect: &Bound<'_, PyAny>) -> PyResult { + if dialect.is_truthy()? { + return dialect.extract(); + } + Ok(GENERIC_DIALECT.to_owned()) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/expression_source_shapes.rs b/crates/sqlbuild-python-project/src/analysis_session/main/expression_source_shapes.rs new file mode 100644 index 0000000000..eb5af66909 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/expression_source_shapes.rs @@ -0,0 +1,14 @@ +//! One shape per expression, cached on `profile`'s binding catalog; a failure raises. + +use crate::analysis_session::_helpers::stage_functions; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyTuple; + +/// One shape per expression, cached on `profile`'s binding catalog; a failure raises. +#[inline] +pub fn expression_source_shapes<'py>( + expressions: &Bound<'py, PyAny>, + profile: &Bound<'py, PyAny>, +) -> PyResult> { + stage_functions::expression_source_shapes(expressions, profile) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_expression_source_shapes.rs b/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_expression_source_shapes.rs new file mode 100644 index 0000000000..203ec0e8f8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_expression_source_shapes.rs @@ -0,0 +1,17 @@ +//! One shape per expression, cached on the profile's binding catalog; a failure raises. + +use crate::analysis_session::_helpers::stage_functions::expression_source_shapes; +use pyo3::prelude::{Bound, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_python_base::boundary::models::Request; + +/// One shape per expression, cached on the profile's binding catalog; a failure raises. +#[pyfunction] +#[pyo3(signature = (**request))] +pub fn infer_native_expression_source_shapes<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("infer_native_expression_source_shapes", request)?; + expression_source_shapes(&request.get("expressions")?, &request.get("profile")?) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_query_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_query_columns.rs new file mode 100644 index 0000000000..42e356a67a --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/infer_native_query_columns.rs @@ -0,0 +1,67 @@ +//! Each query's columns as `NativeQueryColumns`; a native internal failure raises. + +use crate::analysis_session::_helpers::stage_functions::or_empty; +use crate::analysis_session::_helpers::stage_functions::query_column_rows; +use crate::analysis_session::types::ColumnsRow; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_python_base::boundary::models::Request; +use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; + +/// Each query's columns as `NativeQueryColumns`; a native internal failure raises. +#[pyfunction] +#[pyo3(signature = (**request))] +pub fn infer_native_query_columns<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("infer_native_query_columns", request)?; + let py: Python<'py> = request.py(); + let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); + let types: Bound<'py, PyAny> = or_empty(request.optional("column_types_by_table")?, &empty)?; + let nullability: Bound<'py, PyAny> = + or_empty(request.optional("column_nullability_by_table")?, &empty)?; + let rows: Vec = query_column_rows( + &request.get("queries")?, + &request.get("profile")?, + (&types, &nullability), + )?; + let mut analyses: Vec> = Vec::with_capacity(rows.len()); + for (succeeded, columns, has_star) in rows { + let columns: Bound<'py, PyAny> = match columns { + None => py.None().into_bound(py), + Some(columns) => { + let mut built: Vec> = Vec::with_capacity(columns.len()); + for (name, data_type, nullability) in columns { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("name", name)?; + arguments.set_item("type", data_type)?; + arguments.set_item( + "nullability", + enum_from_value(py, RecordId::InferredNullability, nullability)?, + )?; + built.push(new_record( + py, + RecordId::InferredColumn, + (), + Some(&arguments), + )?); + } + PyTuple::new(py, built)?.into_any() + } + }; + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("succeeded", succeeded)?; + arguments.set_item("columns", columns)?; + arguments.set_item("has_star", has_star)?; + analyses.push(new_record( + py, + RecordId::NativeQueryColumns, + (), + Some(&arguments), + )?); + } + PyTuple::new(py, analyses) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/mod.rs b/crates/sqlbuild-python-project/src/analysis_session/main/mod.rs new file mode 100644 index 0000000000..7d47127629 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/mod.rs @@ -0,0 +1,18 @@ +pub mod case_sensitive_shapes; +pub mod dialect_or_generic; +pub mod expression_source_shapes; +pub mod infer_native_expression_source_shapes; +pub mod infer_native_query_columns; +pub mod nullability_rules; +pub mod profile_dialect; +pub mod prove_dynamic_column_contracts; +pub mod query_column_rows; +pub mod query_columns_error; +pub mod register_fixture_columns; +pub mod register_query_columns; +pub mod register_session; +pub mod register_stage_functions; +pub mod return_types; +pub mod shape_rows; +pub mod start_model_analysis_session; +pub mod text_pairs; diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/nullability_rules.rs b/crates/sqlbuild-python-project/src/analysis_session/main/nullability_rules.rs new file mode 100644 index 0000000000..439f65fec8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/nullability_rules.rs @@ -0,0 +1,42 @@ +//! The adapter nullability rules as `(name, rule id)` rows, and the adapter's own rules. + +use crate::analysis_session::_helpers::profile_inputs::AdapterRules; +use crate::analysis_session::_helpers::profile_inputs::KNOWN_RULES; +use crate::analysis_session::constants::ADAPTER_RULE_ID; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +/// The adapter nullability rules as `(name, rule id)` rows, and the adapter's own rules. +pub fn nullability_rules(profile: &Bound<'_, PyAny>) -> PyResult<(Pairs, AdapterRules)> { + let py: Python<'_> = profile.py(); + let known: Bound<'_, PyDict> = PyDict::new(py); + for (rule, id) in KNOWN_RULES { + known.set_item(host_entry(py, rule)?, id)?; + } + let adapter: Bound<'_, PyDict> = PyDict::new(py); + let mut rows: Pairs = Vec::new(); + for item in native_field(profile, "function_nullability_rules")? + .call_method0("items")? + .try_iter()? + { + let (name, rule): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + let id: String = match known.get_item(&rule)? { + Some(id) => id.extract()?, + None => { + adapter.set_item(&name, &rule)?; + ADAPTER_RULE_ID.to_owned() + } + }; + rows.push((name.extract()?, id)); + } + if adapter.is_empty() { + return Ok((rows, None)); + } + let nullability: Bound<'_, PyAny> = record_class(py, RecordId::InferredNullability)?; + Ok((rows, Some((adapter.unbind(), nullability.unbind())))) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/profile_dialect.rs b/crates/sqlbuild-python-project/src/analysis_session/main/profile_dialect.rs new file mode 100644 index 0000000000..0004287fe3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/profile_dialect.rs @@ -0,0 +1,10 @@ +//! `profile.sql_analysis_dialect or "generic"`. + +use crate::analysis_session::main::dialect_or_generic::dialect_or_generic; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// `profile.sql_analysis_dialect or "generic"`. +pub fn profile_dialect(profile: &Bound<'_, PyAny>) -> PyResult { + dialect_or_generic(&native_field(profile, "sql_analysis_dialect")?) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/prove_dynamic_column_contracts.rs b/crates/sqlbuild-python-project/src/analysis_session/main/prove_dynamic_column_contracts.rs new file mode 100644 index 0000000000..434f9c01e9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/prove_dynamic_column_contracts.rs @@ -0,0 +1,45 @@ +//! Each model's dynamic pivot proof outside any session. + +use crate::analysis_session::_helpers::session::ContractRow; +use crate::analysis_session::_helpers::session::contract_row; +use crate::analysis_session::_helpers::session::families_by_table; +use crate::analysis_session::_helpers::session::internal_error; +use crate::analysis_session::_helpers::session::pivot_model; +use crate::analysis_session::models::PivotRequestRow; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_analysis::assembly::analysis_session::main::prove_dynamic_contracts::prove_dynamic_contracts; +use sqlbuild_analysis::assembly::analysis_session::models::{ + PivotBatchRequest, PivotOutcome, PivotTables, +}; +use sqlbuild_python_base::types::CompilerDetach; + +/// Each model's dynamic pivot proof outside any session. +#[pyfunction] +pub fn prove_dynamic_column_contracts( + py: Python<'_>, + request: PivotRequestRow, +) -> PyResult> { + let PivotRequestRow { + dialect, + column_types, + authoritative_types, + column_nullability, + families_by_table: by_table, + models, + } = request; + let request = PivotBatchRequest { + tables: PivotTables { + dialect, + column_types, + authoritative_types, + column_nullability, + families_by_table: families_by_table(by_table), + }, + models: models.into_iter().map(pivot_model).collect(), + }; + let outcomes: Vec = py + .compiler_detach(|| prove_dynamic_contracts(&request)) + .map_err(|reason| internal_error(&reason))?; + Ok(outcomes.into_iter().map(contract_row).collect()) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/query_column_rows.rs b/crates/sqlbuild-python-project/src/analysis_session/main/query_column_rows.rs new file mode 100644 index 0000000000..7695575ae0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/query_column_rows.rs @@ -0,0 +1,15 @@ +//! Query column rows. + +use crate::analysis_session::_helpers::stage_functions; +use crate::analysis_session::types::ColumnsRow; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Query column rows. +#[inline] +pub fn query_column_rows( + queries: &Bound<'_, PyAny>, + profile: &Bound<'_, PyAny>, + (types, nullability): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult> { + stage_functions::query_column_rows(queries, profile, (types, nullability)) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/query_columns_error.rs b/crates/sqlbuild-python-project/src/analysis_session/main/query_columns_error.rs new file mode 100644 index 0000000000..830d37bf47 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/query_columns_error.rs @@ -0,0 +1,17 @@ +//! A normalization rejection as Python's `ValueError`; anything else as an internal failure. + +use crate::analysis_session::_helpers::query_columns::QUERY_COLUMNS_CONTEXT; +use pyo3::PyErr; +use sqlbuild_analysis::assembly::analysis_session::errors::QueryColumnsFailure; +use sqlbuild_core::panics::main::native_failure::native_failure; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; + +/// A normalization rejection as Python's `ValueError`; anything else as an internal failure. +pub fn query_columns_error(failure: QueryColumnsFailure) -> PyErr { + match failure { + QueryColumnsFailure::Normalization(message) => compiler_error(message), + QueryColumnsFailure::Internal(reason) => { + compiler_error(native_failure(QUERY_COLUMNS_CONTEXT, &reason)) + } + } +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/register_fixture_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/main/register_fixture_columns.rs new file mode 100644 index 0000000000..e80dec0138 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/register_fixture_columns.rs @@ -0,0 +1,12 @@ +//! Register the fixture columns functions and classes. + +use crate::analysis_session::_helpers::fixture_columns::fixture_inference_count; +use crate::analysis_session::_helpers::fixture_columns::infer_fixture_column_facts; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the fixture columns functions and classes. +pub fn register_fixture_columns(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(infer_fixture_column_facts, module)?)?; + module.add_function(wrap_pyfunction!(fixture_inference_count, module)?) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/register_query_columns.rs b/crates/sqlbuild-python-project/src/analysis_session/main/register_query_columns.rs new file mode 100644 index 0000000000..b6aee381b3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/register_query_columns.rs @@ -0,0 +1,11 @@ +//! Register the query columns functions and classes. + +use crate::analysis_session::_helpers::query_columns::fixture_projection_facts; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the query columns functions and classes. +pub fn register_query_columns(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(fixture_projection_facts, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/register_session.rs b/crates/sqlbuild-python-project/src/analysis_session/main/register_session.rs new file mode 100644 index 0000000000..175ff571d5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/register_session.rs @@ -0,0 +1,10 @@ +//! Register the session functions and classes. + +use crate::analysis_session::models::NativeModelAnalysisSession; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the session functions and classes. +pub fn register_session(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs b/crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs new file mode 100644 index 0000000000..8ccb4adce7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs @@ -0,0 +1,20 @@ +//! Register the stage functions functions and classes. + +use crate::analysis_session::_helpers::stage_functions::infer_native_query_column_rows; +use crate::analysis_session::_helpers::stage_functions::prove_native_dynamic_contracts; +use crate::analysis_session::main::infer_native_expression_source_shapes::infer_native_expression_source_shapes; +use crate::analysis_session::main::infer_native_query_columns::infer_native_query_columns; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the stage functions functions and classes. +pub fn register_stage_functions(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(infer_native_query_column_rows, module)?)?; + module.add_function(wrap_pyfunction!(infer_native_query_columns, module)?)?; + module.add_function(wrap_pyfunction!( + infer_native_expression_source_shapes, + module + )?)?; + module.add_function(wrap_pyfunction!(prove_native_dynamic_contracts, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/return_types.rs b/crates/sqlbuild-python-project/src/analysis_session/main/return_types.rs new file mode 100644 index 0000000000..2f78eb422a --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/return_types.rs @@ -0,0 +1,11 @@ +//! `profile.function_return_types` items in order. + +use crate::analysis_session::_helpers::profile_inputs::pairs; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// `profile.function_return_types` items in order. +pub fn return_types(profile: &Bound<'_, PyAny>) -> PyResult { + pairs(&native_field(profile, "function_return_types")?) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs b/crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs new file mode 100644 index 0000000000..3772c68e18 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs @@ -0,0 +1,16 @@ +//! `{relation: {column: value}}` as ordered rows, each value as `str(value)`. + +use crate::analysis_session::main::text_pairs::text_pairs; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::analysis_session::types::Shapes; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; + +/// `{relation: {column: value}}` as ordered rows, each value as `str(value)`. +pub fn shape_rows(shapes: &Bound<'_, PyAny>) -> PyResult { + let mut rows: Shapes = Vec::new(); + for item in mapping_items(shapes)?.try_iter()? { + let (name, shape): (String, Bound<'_, PyAny>) = item?.extract()?; + rows.push((name, text_pairs(&shape)?)); + } + Ok(rows) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/start_model_analysis_session.rs b/crates/sqlbuild-python-project/src/analysis_session/main/start_model_analysis_session.rs new file mode 100644 index 0000000000..7340e2cd0a --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/start_model_analysis_session.rs @@ -0,0 +1,71 @@ +//! Start a session on `catalog`. + +use crate::analysis::models::ProjectCatalog; +use crate::analysis_session::_helpers::nullability_rules::{RuleFailure, nullability_callback}; +use crate::analysis_session::_helpers::session::MODEL_ANALYSIS_STORE_KIND; +use crate::analysis_session::_helpers::session::internal_error; +use crate::analysis_session::_helpers::session::session_request; +use crate::analysis_session::models::CacheStatsRow; +use crate::analysis_session::models::NativeModelAnalysisSession; +use crate::analysis_session::models::RequestRow; +use pyo3::prelude::{Py, PyAny, PyRef, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_analysis::assembly::analysis_session::main::attach_analysis_cache::attach_analysis_cache; +use sqlbuild_analysis::assembly::analysis_session::main::start_analysis_session::start_analysis_session; +use sqlbuild_analysis::assembly::analysis_session::models::{AnalysisSession, SessionRequest}; +use sqlbuild_python_base::cache::main::open_store::open_store; +use sqlbuild_python_base::types::CompilerDetach; +use std::path::PathBuf; + +/// Start a session on `catalog`. +pub fn start_model_analysis_session( + catalog: PyRef<'_, ProjectCatalog>, + request: RequestRow, + cache: Option<(PathBuf, String)>, + adapter_rules: Option<(Py, Py)>, +) -> PyResult { + let py: Python<'_> = catalog.py(); + let mut request: SessionRequest = + sqlbuild_python_base::compile_context::main::delta::inject_relations::inject_relations( + session_request(request), + )?; + let rule_failure: RuleFailure = RuleFailure::default(); + let dialect: String = request.dialect.clone(); + request.nullability_callback = adapter_rules + .map(|(rules, nullability)| nullability_callback(rules, nullability, rule_failure.clone())); + let catalog = &catalog.inner; + let mut session: AnalysisSession = py + .compiler_detach(|| start_analysis_session(request, catalog)) + .map_err(|reason| internal_error(&reason))?; + let cache: Option<(PathBuf, String)> = cache.filter(|_| { + sqlbuild_python_base::compile_context::main::delta::injection::injection().is_none() + }); + let mut cache_path: Option = None; + let mut cache_stats: Option = None; + if let Some((path, environment)) = cache { + match open_store(py, &path, MODEL_ANALYSIS_STORE_KIND, &environment) { + Ok(store) => { + attach_analysis_cache(&mut session, store); + cache_path = Some(path); + } + Err(error) => { + cache_stats = Some(CacheStatsRow { + hits: 0, + misses: 0, + stored: 0, + failure: Some(error.to_string()), + }); + } + } + } + Ok(NativeModelAnalysisSession { + inner: Some(session), + finished: None, + outcomes: Vec::new(), + sharing: (0, 0), + cache_path, + cache_stats, + rule_failure, + dialect, + }) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/main/text_pairs.rs b/crates/sqlbuild-python-project/src/analysis_session/main/text_pairs.rs new file mode 100644 index 0000000000..2965be6b94 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/main/text_pairs.rs @@ -0,0 +1,15 @@ +//! `{column: value}` as ordered pairs, each value as `str(value)`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; + +/// `{column: value}` as ordered pairs, each value as `str(value)`. +pub fn text_pairs(shape: &Bound<'_, PyAny>) -> PyResult { + let mut pairs: Pairs = Vec::new(); + for column in mapping_items(shape)?.try_iter()? { + let (column, value): (String, Bound<'_, PyAny>) = column?.extract()?; + pairs.push((column, value.str()?.extract()?)); + } + Ok(pairs) +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/mod.rs b/crates/sqlbuild-python-project/src/analysis_session/mod.rs new file mode 100644 index 0000000000..6342b807d8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/mod.rs @@ -0,0 +1,7 @@ +//! The analysis session: stage functions, query columns, nullability rules, profile inputs and fixture inferences. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/analysis_session/models.rs b/crates/sqlbuild-python-project/src/analysis_session/models.rs new file mode 100644 index 0000000000..2180433bb3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/models.rs @@ -0,0 +1,452 @@ +//! Public records of analysis session. + +use crate::analysis::models::Positional; +use crate::analysis_session::_helpers::fixture_columns::infer_fixture_column_facts; +use crate::analysis_session::_helpers::nullability_rules::RuleFailure; +use crate::analysis_session::_helpers::nullability_rules::raised; +use crate::analysis_session::_helpers::session::ColumnRow; +use crate::analysis_session::_helpers::session::ContractRow; +use crate::analysis_session::_helpers::session::FinishRow; +use crate::analysis_session::_helpers::session::OutcomeRow; +use crate::analysis_session::_helpers::session::PivotModelRow; +use crate::analysis_session::_helpers::session::StepRow; +use crate::analysis_session::_helpers::session::contract_row; +use crate::analysis_session::_helpers::session::internal_error; +use crate::analysis_session::_helpers::session::outcome_row; +use crate::analysis_session::_helpers::session::pivot_model; +use crate::analysis_session::_helpers::session::step_row; +use crate::analysis_session::types::PivotRequestTuple; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use sqlbuild_analysis::assembly::analysis_session::main::finish_analysis_session::finish_analysis_session; +use sqlbuild_analysis::assembly::analysis_session::main::finished_fact_models::finished_fact_models; +use sqlbuild_analysis::assembly::analysis_session::main::prove_finished_dynamic_contracts::prove_finished_dynamic_contracts; +use sqlbuild_analysis::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql; +use sqlbuild_analysis::assembly::analysis_session::main::run_analysis_session::run_analysis_session; +use sqlbuild_analysis::assembly::analysis_session::main::session_sharing::session_sharing; +use sqlbuild_analysis::assembly::analysis_session::main::take_analysis_cache::take_analysis_cache; +use sqlbuild_analysis::assembly::analysis_session::models::AnalysisCacheStats; +use sqlbuild_analysis::assembly::analysis_session::models::AnalysisSession; +use sqlbuild_analysis::assembly::analysis_session::models::FinishedSession; +use sqlbuild_analysis::assembly::analysis_session::models::ModelOutcome; +use sqlbuild_analysis::assembly::analysis_session::models::PivotModel; +use sqlbuild_analysis::assembly::analysis_session::models::PivotOutcome; +use sqlbuild_analysis::assembly::analysis_session::models::SessionOutcome; +use sqlbuild_analysis::assembly::analysis_session::models::SessionStep; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; +use sqlbuild_analysis::assembly::analysis_session::types::Shapes; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; +use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; +use sqlbuild_cache::store::models::NativeStore; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; +use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::cache::main::save_store::save_store; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::types::CompilerDetach; +use std::path::PathBuf; +use std::sync::Mutex; + +/// Reuse inference for identical fixture SQL under an equal, catalog-free profile. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug, Default)] +pub struct FixtureColumnInferences { + pub(crate) by_profile: Mutex, Py)>>, +} + +#[pymethods] +impl FixtureColumnInferences { + #[new] + pub fn new() -> Self { + Self { + by_profile: Mutex::new(Vec::new()), + } + } + + /// The inference for one fixture query, computing each distinct query once. + #[pyo3(signature = (*, query_sql, inference_profile))] + pub(crate) fn infer<'py>( + &self, + query_sql: &Bound<'py, PyAny>, + inference_profile: &Bound<'py, PyAny>, + ) -> PyResult> { + let py = query_sql.py(); + let compute = || { + infer_fixture_column_facts(Some(&keywords( + py, + &[ + ("query_sql", query_sql.clone()), + ("inference_profile", inference_profile.clone()), + ], + )?)) + }; + if !native_field(inference_profile, "binding_catalog")?.is_none() { + return compute(); + } + let inferred = self.profile_results(py, inference_profile)?; + if let Some(found) = inferred.get_item(query_sql)? { + return Ok(found); + } + let computed = compute()?; + inferred.set_item(query_sql, &computed)?; + Ok(computed) + } +} + +impl FixtureColumnInferences { + pub(crate) fn profile_results<'py>( + &self, + py: Python<'py>, + inference_profile: &Bound<'py, PyAny>, + ) -> PyResult> { + let known: Vec<(Py, Py)> = self + .by_profile + .lock() + .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err("fixture inference cache"))? + .iter() + .map(|(profile, results)| (profile.clone_ref(py), results.clone_ref(py))) + .collect(); + for (profile, results) in known { + if profile.bind(py).eq(inference_profile)? { + return Ok(results.into_bound(py)); + } + } + let results = PyDict::new(py); + self.by_profile + .lock() + .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err("fixture inference cache"))? + .push((inference_profile.clone().unbind(), results.clone().unbind())); + Ok(results) + } +} + +/// Fixture column inferences computed since the last reset. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct FixtureInferenceCount { + pub computed: u64, +} + +/// One declared dynamic column family. +#[derive(Debug)] +pub struct FamilyRow { + pub name: String, + pub pivot_column: String, + pub value_column: String, + pub aggregate: String, + pub data_type: String, + pub name_pattern: Option, +} + +impl<'py> pyo3::FromPyObject<'py> for FamilyRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (name, pivot_column, value_column, aggregate, data_type, name_pattern): ( + String, + String, + String, + String, + String, + Option, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + name, + pivot_column, + value_column, + aggregate, + data_type, + name_pattern, + }) + } +} + +impl From for FamilyRow { + fn from(family: sqlbuild_model_config::header_metadata::models::DynamicFamily) -> Self { + Self { + name: family.name, + pivot_column: family.pivot_column, + value_column: family.value_column, + aggregate: family.aggregate, + data_type: family.data_type, + name_pattern: family.name_pattern, + } + } +} + +/// A model's name, SQL, placeholders, references, lineage, required names, flags and pivot. +#[derive(Debug)] +pub struct ModelRow { + pub name: String, + pub query_sql: String, + pub placeholders: Pairs, + pub reference_rows: Vec<(String, String)>, + pub recover_cte_facts: bool, + pub snapshot_columns: Option<(String, String)>, + pub dynamic_families: Vec, + pub sql_macro_checks: Vec, +} + +/// The profile settings, relation shapes and models one session analyses. +#[derive(Debug)] +pub struct RequestRow { + pub dialect: String, + pub case_sensitive_shapes: bool, + pub quoted_ignore_case: bool, + pub function_return_types: Pairs, + pub known_functions: Vec, + pub nullability_rules: Option, + pub rich_type_inference: bool, + pub column_types: Shapes, + pub column_nullability: Shapes, + pub complete_schemas: Shapes, + pub catalog_schemas: Shapes, + pub dynamic_families_by_table: Vec<(String, Vec)>, + pub models: Vec, +} + +/// One dynamic pivot proof. +#[derive(Debug)] +pub struct ProofRow { + pub output_proven: bool, + pub fixed_columns: Vec, + /// `(family, inferred type)` of each family. + pub families: Vec<(String, Option)>, + pub input_relations: Vec, + pub failure_reason: Option, + pub bare_dynamic_pivot: bool, +} + +impl<'py> pyo3::IntoPyObject<'py> for ProofRow { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject( + ( + self.output_proven, + self.fixed_columns, + self.families, + self.input_relations, + self.failure_reason, + self.bare_dynamic_pivot, + ), + py, + ) + } +} + +/// The dialect, relation facts and families by table every proof reads, and the models. +#[derive(Debug)] +pub struct PivotRequestRow { + pub dialect: String, + pub column_types: Shapes, + pub authoritative_types: Shapes, + pub column_nullability: Shapes, + pub families_by_table: Vec<(String, Vec)>, + pub models: Vec, +} + +impl<'py> pyo3::FromPyObject<'py> for PivotRequestRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let ( + dialect, + column_types, + authoritative_types, + column_nullability, + families_by_table, + models, + ): PivotRequestTuple = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + dialect, + column_types, + authoritative_types, + column_nullability, + families_by_table, + models, + }) + } +} + +/// The analysis cache's hits, misses, stored entries and why it was not read or saved. +#[derive(Clone, Debug)] +pub struct CacheStatsRow { + pub hits: usize, + pub misses: usize, + pub stored: usize, + pub failure: Option, +} + +impl<'py> pyo3::IntoPyObject<'py> for CacheStatsRow { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject((self.hits, self.misses, self.stored, self.failure), py) + } +} + +/// One compile's native model analysis; an internal failure raises `NativeCompilerError`. +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug)] +pub struct NativeModelAnalysisSession { + pub(crate) inner: Option, + pub(crate) finished: Option, + /// Each requested model's outcome and dynamic pivot proof, in request order, once finished. + pub(crate) outcomes: Vec<(ModelOutcome, PivotOutcome)>, + pub(crate) sharing: (usize, usize), + pub(crate) cache_path: Option, + pub(crate) cache_stats: Option, + pub(crate) rule_failure: RuleFailure, + /// The analysis dialect the session's models were marked and analysed in. + pub(crate) dialect: String, +} + +impl NativeModelAnalysisSession { + /// The finished session later compile stages read, once `finish` has succeeded. + pub fn finished_session(&self) -> Option<&FinishedSession> { + self.finished.as_ref() + } + + /// Each requested model's finished outcome and dynamic pivot proof, in request order. + pub fn outcomes(&self) -> &[(ModelOutcome, PivotOutcome)] { + &self.outcomes + } + + /// The finished outcome and dynamic pivot proof of the model requested at `index`. + pub fn outcome(&self, index: usize) -> PyResult<&(ModelOutcome, PivotOutcome)> { + self.outcomes + .get(index) + .ok_or_else(|| internal_error("the session returned no analysis for a model")) + } + + /// The running session, or `NativeCompilerError` once it has failed or finished. + pub fn dialect(&self) -> &str { + &self.dialect + } + + pub(crate) fn running(&mut self) -> PyResult { + self.inner + .take() + .ok_or_else(|| internal_error("the session is not running")) + } +} + +impl NativeModelAnalysisSession { + /// Analyse every model. + pub fn run(&mut self, py: Python<'_>) -> PyResult { + let mut session: AnalysisSession = self.running()?; + let result: Result<(AnalysisSession, SessionStep), String> = + py.compiler_detach(move || { + run_analysis_session(&mut session).map(|step| (session, step)) + }); + if let Some(error) = raised(&self.rule_failure) { + return Err(error); + } + let (session, step) = result.map_err(|reason| internal_error(&reason))?; + self.sharing = session_sharing(&session); + self.inner = Some(session); + Ok(step_row(step)) + } + + /// Every model's outcome; saves the cache best-effort. + pub fn finish(&mut self, py: Python<'_>) -> PyResult { + let mut session: AnalysisSession = self.running()?; + let cache: Option<(NativeStore, AnalysisCacheStats)> = take_analysis_cache(&mut session); + let (outcome, finished) = catch_compiler_panic(|| finish_analysis_session(session)) + .map_err(|reason| internal_error(&reason))?; + self.finished = Some(finished); + if let (Some((store, stats)), Some(path)) = (cache, self.cache_path.as_ref()) { + let saved: Option = save_store(py, &store, path, &[]) + .err() + .map(|error| error.to_string()); + self.cache_stats = Some(CacheStatsRow { + hits: stats.hits, + misses: stats.misses, + stored: stats.stored, + failure: saved, + }); + } + let SessionOutcome { + models, + schema_additions, + analysis_names, + dynamic_contracts, + } = outcome; + let succeeded: Vec = models + .iter() + .map(|model| model.analysis.analysis_succeeded) + .collect(); + self.outcomes = models.into_iter().zip(dynamic_contracts).collect(); + Ok((schema_additions, analysis_names, succeeded)) + } +} + +#[pymethods] +impl NativeModelAnalysisSession { + /// Models whose output names and lineage the finished session keeps for later stages. + #[getter] + pub fn fact_models(&self) -> Vec { + self.finished + .as_ref() + .map(finished_fact_models) + .unwrap_or_default() + } + + /// Each requested model's raw binding diagnostic rows, in request order. + pub fn binding_rows(&self) -> Vec>> { + self.outcomes + .iter() + .map(|(outcome, _)| Positional::wrap_all(outcome.analysis.binding_diagnostics.clone())) + .collect() + } + + /// The model at `index`'s outcome and proof rows, for hand-built model inputs only. + pub(crate) fn outcome_rows(&self, index: usize) -> PyResult<(OutcomeRow, ContractRow)> { + let (outcome, contract) = self.outcome(index)?; + let cleaned: String = resolved_cleaned_sql(outcome) + .map_err(compiler_error)? + .into_owned(); + Ok(( + outcome_row(outcome.clone(), cleaned), + contract_row(contract.clone()), + )) + } + + /// The analysis cache's `(hits, misses, stored, failure)`; None without a cache. + #[getter] + pub fn cache_stats(&self) -> Option { + self.cache_stats.clone() + } + + /// Each model's dynamic pivot proof from the finished session's tables. + pub fn prove_dynamic_contracts( + &self, + py: Python<'_>, + models: Vec, + ) -> PyResult> { + let session: &FinishedSession = self + .finished + .as_ref() + .ok_or_else(|| internal_error("the session has not finished"))?; + let models: Vec = models.into_iter().map(pivot_model).collect(); + let outcomes: Vec = py + .compiler_detach(|| prove_finished_dynamic_contracts(session, &models)) + .map_err(|reason| internal_error(&reason))?; + Ok(outcomes.into_iter().map(contract_row).collect()) + } + + /// `(shared members, shared members re-analysed alone)` as of the last step. + #[getter] + pub fn sharing(&self) -> (usize, usize) { + self.sharing + } +} diff --git a/crates/sqlbuild-python-project/src/analysis_session/types.rs b/crates/sqlbuild-python-project/src/analysis_session/types.rs new file mode 100644 index 0000000000..8df6a0949c --- /dev/null +++ b/crates/sqlbuild-python-project/src/analysis_session/types.rs @@ -0,0 +1,18 @@ +//! Public traits and type aliases of analysis session. + +use crate::analysis_session::_helpers::session::PivotModelRow; +use crate::analysis_session::models::FamilyRow; +use sqlbuild_analysis::assembly::analysis_session::types::Shapes; + +/// `(succeeded, [(name, type, nullability)] or None, has star)`. +pub type ColumnsRow = (bool, Option, String)>>, bool); + +/// The tuple Python sends for a `PivotRequestRow`, in field order. +pub type PivotRequestTuple = ( + String, + Shapes, + Shapes, + Shapes, + Vec<(String, Vec)>, + Vec, +); diff --git a/crates/sqlbuild-python-project/src/auditing/_helpers/mod.rs b/crates/sqlbuild-python-project/src/auditing/_helpers/mod.rs new file mode 100644 index 0000000000..14f3a4214d --- /dev/null +++ b/crates/sqlbuild-python-project/src/auditing/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod render; diff --git a/crates/sqlbuild-python-project/src/auditing/_helpers/render.rs b/crates/sqlbuild-python-project/src/auditing/_helpers/render.rs new file mode 100644 index 0000000000..830e89296f --- /dev/null +++ b/crates/sqlbuild-python-project/src/auditing/_helpers/render.rs @@ -0,0 +1,31 @@ +//! Audit SQL with its reference calls replaced by relation names or overrides. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; + +pub(crate) const REF_FUNCTION: &str = "__ref"; +pub(crate) const SEED_FUNCTION: &str = "__seed"; +pub(crate) const SOURCE_FUNCTION: &str = "__source"; + +/// `mapping.get(name)`, or `None` when absent or `None`. +pub(crate) fn lookup<'py>( + mapping: &Bound<'py, PyAny>, + name: &str, +) -> PyResult>> { + let value: Bound<'py, PyAny> = mapping_get(mapping, name)?; + Ok(Some(value).filter(|value| !value.is_none())) +} + +/// The qualified name of the relation location `mapping` holds for `name`, if any. +pub(crate) fn qualified<'py>( + mapping: &Bound<'py, PyAny>, + name: &str, +) -> PyResult>> { + let Some(location) = lookup(mapping, name)? else { + return Ok(None); + }; + let qualified: Bound<'py, PyAny> = native_field(&location, "qualified_name")?; + Ok(Some(qualified).filter(|qualified| !qualified.is_none())) +} diff --git a/crates/sqlbuild-python-project/src/auditing/main/mod.rs b/crates/sqlbuild-python-project/src/auditing/main/mod.rs new file mode 100644 index 0000000000..fe7660516e --- /dev/null +++ b/crates/sqlbuild-python-project/src/auditing/main/mod.rs @@ -0,0 +1,2 @@ +pub mod register_render; +pub mod render_audit_sql; diff --git a/crates/sqlbuild-python-project/src/auditing/main/register_render.rs b/crates/sqlbuild-python-project/src/auditing/main/register_render.rs new file mode 100644 index 0000000000..96a1f2dc26 --- /dev/null +++ b/crates/sqlbuild-python-project/src/auditing/main/register_render.rs @@ -0,0 +1,11 @@ +//! Register the render functions and classes. + +use crate::auditing::main::render_audit_sql::render_audit_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the render functions and classes. +pub fn register_render(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_audit_sql, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs b/crates/sqlbuild-python-project/src/auditing/main/render_audit_sql.rs similarity index 55% rename from crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs rename to crates/sqlbuild-python-project/src/auditing/main/render_audit_sql.rs index 9bff3a370d..fda991b3d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs +++ b/crates/sqlbuild-python-project/src/auditing/main/render_audit_sql.rs @@ -1,40 +1,22 @@ -//! Audit SQL with its reference calls replaced by relation names or overrides. +//! Render audit SQL from unresolved markers, applying relation overrides first. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use crate::auditing::_helpers::render::REF_FUNCTION; +use crate::auditing::_helpers::render::SEED_FUNCTION; +use crate::auditing::_helpers::render::SOURCE_FUNCTION; +use crate::auditing::_helpers::render::lookup; +use crate::auditing::_helpers::render::qualified; +use crate::references::main::render_source_read::render_source_read; +use crate::references::main::render_source_relation::render_source_relation; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::auditing::main::replace_reference_calls::replace_reference_calls; - -use crate::bindings::_helpers::references::source_relations::{ - render_source_read, render_source_relation, -}; use sqlbuild_python_base::boundary::models::Request; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; - -const REF_FUNCTION: &str = "__ref"; -const SEED_FUNCTION: &str = "__seed"; -const SOURCE_FUNCTION: &str = "__source"; - -/// `mapping.get(name)`, or `None` when absent or `None`. -fn lookup<'py>(mapping: &Bound<'py, PyAny>, name: &str) -> PyResult>> { - let value: Bound<'py, PyAny> = mapping_get(mapping, name)?; - Ok(Some(value).filter(|value| !value.is_none())) -} - -/// The qualified name of the relation location `mapping` holds for `name`, if any. -fn qualified<'py>(mapping: &Bound<'py, PyAny>, name: &str) -> PyResult>> { - let Some(location) = lookup(mapping, name)? else { - return Ok(None); - }; - let qualified: Bound<'py, PyAny> = native_field(&location, "qualified_name")?; - Ok(Some(qualified).filter(|qualified| !qualified.is_none())) -} /// Render audit SQL from unresolved markers, applying relation overrides first. #[pyfunction] #[pyo3(signature = (**request))] -pub(crate) fn render_audit_sql(request: Option<&Bound<'_, PyDict>>) -> PyResult { +pub fn render_audit_sql(request: Option<&Bound<'_, PyDict>>) -> PyResult { let request = Request::new("render_audit_sql", request)?; let sql: String = request.get("unresolved_sql")?.extract()?; let models: Bound<'_, PyAny> = request.get("model_locations")?; @@ -73,8 +55,3 @@ pub(crate) fn render_audit_sql(request: Option<&Bound<'_, PyDict>>) -> PyResult< } }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(render_audit_sql, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/auditing/mod.rs b/crates/sqlbuild-python-project/src/auditing/mod.rs new file mode 100644 index 0000000000..877ebc65aa --- /dev/null +++ b/crates/sqlbuild-python-project/src/auditing/mod.rs @@ -0,0 +1,4 @@ +//! Rendering of audit SQL. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs b/crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs new file mode 100644 index 0000000000..e0a418a5c9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs @@ -0,0 +1,54 @@ +//! A compiled model's authored cursor inputs resolved into filter and watermark roles. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; + +pub(crate) const CURSOR_INPUTS_KEY: &str = "cursor_inputs"; +const FILTER_ROLE: &str = "filter"; +const WATERMARK_ROLE: &str = "watermark"; +pub(crate) const WATERMARK_STRATEGY: &str = "watermark"; +pub(crate) const ROLLING_WINDOW_STRATEGY: &str = "rolling_window"; + +/// Watermark-strategy cursor inputs: each relation's column under the roles it declares. +pub(crate) fn role_blocks<'py>( + authored: &Bound<'py, PyDict>, + (filter, watermark): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), +) -> PyResult<()> { + for (relation, block) in authored.iter() { + let (true, Ok(block)) = ( + relation.is_instance_of::(), + block.downcast::(), + ) else { + continue; + }; + let column: Option> = block.get_item("column")?; + let roles: Option> = block.get_item("roles")?; + let (Some(column), Some(roles)) = (column, roles) else { + continue; + }; + let (true, Ok(roles)) = ( + column.is_instance_of::(), + roles.downcast::(), + ) else { + continue; + }; + if roles.contains(FILTER_ROLE)? { + filter.set_item(&relation, &column)?; + } + if roles.contains(WATERMARK_ROLE)? { + watermark.set_item(&relation, &column)?; + } + } + Ok(()) +} + +/// The string-to-string entries of an authored mapping. +pub(crate) fn string_map<'py>(authored: &Bound<'py, PyDict>) -> PyResult> { + let result: Bound<'py, PyDict> = PyDict::new(authored.py()); + for (key, item) in authored.iter() { + if key.is_instance_of::() && item.is_instance_of::() { + result.set_item(key, item)?; + } + } + Ok(result) +} diff --git a/crates/sqlbuild-python-project/src/compile_attachment/_helpers/mod.rs b/crates/sqlbuild-python-project/src/compile_attachment/_helpers/mod.rs new file mode 100644 index 0000000000..a3f498da74 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compile_attachment/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod cursor_roles; diff --git a/crates/sqlbuild-python-project/src/compile_attachment/main/mod.rs b/crates/sqlbuild-python-project/src/compile_attachment/main/mod.rs new file mode 100644 index 0000000000..f8523830d8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compile_attachment/main/mod.rs @@ -0,0 +1,2 @@ +pub mod resolve_cursor_input_roles; +pub mod resolve_cursor_input_roles_function; diff --git a/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles.rs b/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles.rs new file mode 100644 index 0000000000..61edc9df4e --- /dev/null +++ b/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles.rs @@ -0,0 +1,59 @@ +//! Resolve authored cursor input fields into explicit effective roles. + +use crate::compile_attachment::_helpers::cursor_roles::CURSOR_INPUTS_KEY; +use crate::compile_attachment::_helpers::cursor_roles::ROLLING_WINDOW_STRATEGY; +use crate::compile_attachment::_helpers::cursor_roles::WATERMARK_STRATEGY; +use crate::compile_attachment::_helpers::cursor_roles::role_blocks; +use crate::compile_attachment::_helpers::cursor_roles::string_map; +use crate::compiled_views::main::model_inputs::input_reference_rows::input_reference_rows; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use sqlbuild_project::models::ReferenceRow; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_base::records::types::RecordId; + +/// Resolve authored cursor input fields into explicit effective roles. +#[pyfunction] +#[pyo3(signature = (*, model))] +pub fn resolve_cursor_input_roles<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + let values: Bound<'py, PyAny> = native_field(model, "config")?.getattr("values")?; + let authored: Bound<'py, PyAny> = mapping_get(&values, CURSOR_INPUTS_KEY)?; + let strategy: Bound<'py, PyAny> = mapping_get(&values, "microbatch_strategy")?; + let rolling: bool = strategy.eq(ROLLING_WINDOW_STRATEGY)?; + let mut filter: Bound<'py, PyDict> = PyDict::new(py); + let mut watermark: Bound<'py, PyDict> = PyDict::new(py); + match authored.downcast::() { + Ok(authored) if strategy.eq(WATERMARK_STRATEGY)? => { + role_blocks(authored, (&filter, &watermark))?; + } + Ok(authored) => { + filter = string_map(authored)?; + if !rolling { + watermark = filter.clone(); + } + } + Err(_) => { + let cursor: Bound<'py, PyAny> = mapping_get(&values, "cursor")?; + if cursor.is_instance_of::() { + for ReferenceRow { name, .. } in input_reference_rows(model)? { + filter.set_item(name, &cursor)?; + } + } + if !rolling { + watermark = filter.clone(); + } + } + } + new_record_with( + py, + RecordId::CursorInputRoles, + &[ + ("filter_inputs", filter.into_any()), + ("watermark_inputs", watermark.into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles_function.rs b/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles_function.rs new file mode 100644 index 0000000000..d2eddceeb5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles_function.rs @@ -0,0 +1,10 @@ +//! `resolve_cursor_input_roles` as a function of `module`. + +use crate::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; + +/// `resolve_cursor_input_roles` as a function of `module`. +pub fn resolve_cursor_input_roles_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(resolve_cursor_input_roles, module) +} diff --git a/crates/sqlbuild-python-project/src/compile_attachment/mod.rs b/crates/sqlbuild-python-project/src/compile_attachment/mod.rs new file mode 100644 index 0000000000..66b3451082 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compile_attachment/mod.rs @@ -0,0 +1,4 @@ +//! Cursor roles a compile attaches to models. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/attachment_inputs.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/attachment_inputs.rs new file mode 100644 index 0000000000..2792e52147 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/attachment_inputs.rs @@ -0,0 +1,72 @@ +//! The resource rows and settings the compile attaches into the project, and the installed runs. + +use crate::compiled_project::models::AuditFileRow; +use crate::compiled_project::models::AuditSettings; +use crate::compiled_project::models::DiagnosticSettings; +use crate::compiled_project::models::FunctionRow; +use crate::compiled_project::models::FunctionSettings; +use crate::compiled_project::models::ResourceAttachment; +use crate::compiled_project::models::ScenarioRow; +use crate::compiled_project::models::SqlTestSettings; +use crate::compiled_project::models::TestFileRow; +use crate::compiled_project::types::AttachedSqlTests; +use crate::compiled_project::types::ResourceDiagnostics; +use crate::compiled_project::types::SourceRow; +use crate::compiled_project::types::SourceSettings; +use std::sync::OnceLock; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; + +use crate::compiled_project::models::NativeCompiledProject; +use crate::resource_views::models::NativeAuditInput; +use crate::resource_views::models::NativeFunctionInput; +use crate::resource_views::models::NativeSourceInput; + +const ATTACHMENT_UNINSTALLED: &str = "the native module registered no resource attachment"; + +pub(crate) static INSTALLED: OnceLock = OnceLock::new(); + +/// Attaches the project's sources, given their rows and settings. +pub(crate) type AttachSources = fn( + &Bound<'_, NativeCompiledProject>, + Vec, + SourceSettings, +) -> PyResult>; + +/// Attaches the project's SQL, then Python, functions. +pub(crate) type AttachFunctions = fn( + &Bound<'_, NativeCompiledProject>, + (Vec, Vec), + FunctionSettings, +) -> PyResult>; + +/// Reads the project's resource diagnostics. +pub(crate) type ResourceDiagnosticsRun = for<'py> fn( + &NativeCompiledProject, + Python<'py>, + &DiagnosticSettings, +) -> PyResult>; + +/// Attaches the project's SQL test files, then its scenarios. +pub(crate) type AttachSqlTests = fn( + &Bound<'_, NativeCompiledProject>, + Vec, + Vec, + SqlTestSettings, +) -> PyResult; + +/// Attaches the project's audits; returns their views and the built-in shadow warnings. +pub(crate) type AttachAudits = + for<'py> fn( + &Bound<'py, NativeCompiledProject>, + Vec, + AuditSettings, + ) -> PyResult<(Vec, Vec>)>; + +/// The installed attachment runs. +pub(crate) fn resource_attachment() -> PyResult<&'static ResourceAttachment> { + INSTALLED + .get() + .ok_or_else(|| PyRuntimeError::new_err(ATTACHMENT_UNINSTALLED)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs similarity index 66% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs rename to crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs index f9b2367126..9d7502e05a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs @@ -1,6 +1,5 @@ //! The command target's jobs, loaded when a selector names one or `--changed` needs triggered jobs. -use std::borrow::Cow; use std::cell::RefCell; use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; @@ -8,15 +7,12 @@ use std::path::PathBuf; use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{PyResult, Python}; -use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; use sqlbuild_analysis::graph::main::parse_selector::parse_selector; -use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; -use sqlbuild_analysis::graph::models::{JobSelection, ParsedSelector, ProjectGraph}; +use sqlbuild_analysis::graph::models::{JobSelection, ParsedSelector}; use sqlbuild_cli::native_commands::main::effective_target::effective_target; use sqlbuild_discovery::orchestration_files::models::{OrchestrationInventory, RunFile}; -use crate::bindings::_helpers::compiled_project::constants::{ +use crate::compiled_project::constants::{ JOB_SELECTOR_KINDS, SELECTOR_INTERSECTION, SENSOR_RUN_ARGUMENT, }; use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; @@ -53,72 +49,8 @@ pub(crate) fn with_job_scope( result } -/// `graph` with the command's jobs attached when `selectors` name a job, otherwise `graph` itself. -pub(crate) fn graph_with_jobs<'g>( - py: Python<'_>, - graph: &'g ProjectGraph, - selectors: &[String], - also_known: &[&str], -) -> PyResult> { - Ok(match needed_jobs(py, graph, selectors, also_known)? { - Some(catalog) => Cow::Owned(attach_jobs(graph.clone(), catalog)), - None => Cow::Borrowed(graph), - }) -} - -/// The command's jobs when `selectors` name one and `graph` has none attached yet. -pub(crate) fn needed_jobs( - py: Python<'_>, - graph: &ProjectGraph, - selectors: &[String], - also_known: &[&str], -) -> PyResult>> { - if graph.jobs.is_some() || !selectors_need_jobs(graph, selectors, also_known) { - return Ok(None); - } - Ok(job_catalog(py)?.map(|(catalog, _)| catalog)) -} - -/// The command's job and job group names, loaded on first use; empty outside a command. -pub(crate) fn job_names(py: Python<'_>) -> PyResult> { - Ok(job_catalog(py)? - .map(|(catalog, _)| catalog.into_keys().collect()) - .unwrap_or_default()) -} - -/// Each job a schedule or sensor starts with the nodes it refreshes, less those of jobs `selectors` run. -pub(crate) fn triggered_jobs( - py: Python<'_>, - graph: &ProjectGraph, - selectors: &[String], -) -> PyResult)>> { - let Some((catalog, triggered)) = job_catalog(py)? else { - return Ok(Vec::new()); - }; - if triggered.is_empty() { - return Ok(Vec::new()); - } - let running: BTreeSet = selected_jobs(selectors, &catalog); - let mut graph: ProjectGraph = attach_jobs(graph.clone(), catalog); - let mut resolved = graph.jobs.take().unwrap_or_default(); - let mut running_nodes: BTreeSet = BTreeSet::new(); - for name in &running { - if let Some(Ok(keys)) = resolved.get(name) { - running_nodes.extend(keys.iter().cloned()); - } - } - Ok(triggered - .into_iter() - .filter(|name| !running.contains(name)) - .filter_map(|name| match resolved.remove(&name) { - Some(Ok(keys)) => Some((name, &keys - &running_nodes)), - _ => None, - }) - .collect()) -} - /// The jobs `selectors` name, as `job:` or as a bare job name. -fn selected_jobs( +pub(crate) fn selected_jobs( selectors: &[String], catalog: &BTreeMap, ) -> BTreeSet { @@ -138,7 +70,7 @@ fn selected_jobs( jobs } -fn job_catalog(py: Python<'_>) -> PyResult> { +pub(crate) fn job_catalog(py: Python<'_>) -> PyResult> { let Some(scope) = JOB_SCOPE.with(|scope| { scope.borrow().as_ref().map(|scope| match &scope.catalog { Some(catalog) => Ok(catalog.clone()), diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs new file mode 100644 index 0000000000..bf19c152a0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod attachment_inputs; +pub(crate) mod job_catalog; +pub(crate) mod project; +pub(crate) mod schema_facts; +pub(crate) mod values; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs new file mode 100644 index 0000000000..1b8f938370 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs @@ -0,0 +1,52 @@ +//! The compile's natively owned project, which producers record into and consumers read. + +use crate::compiled_project::models::DeclarationPy; +use sqlbuild_python_base::positional_record; +use std::collections::BTreeMap; + +use pyo3::prelude::PyResult; +use serde_json::Value; +use sqlbuild_analysis::compiled_project::models::DeclarationFact; + +use crate::compiled_project::_helpers::values::plain_value; + +pub(crate) const INPUT_CLASSES_UNSET: &str = "no model loop recorded inputs into this project"; +pub(crate) const ATTACHMENT_LOOP_UNSET: &str = + "no model loop was kept to attach this project's resources"; + +pub(crate) const MATERIALIZED_KEY: &str = "materialized"; +pub(crate) const TABLE_MATERIALIZATION: &str = "table"; + +/// `(name, type)`. +pub(crate) type NamedType = (String, Option); + +positional_record! { + extract; + /// A model's name, relative path, compiled and authored queries, and authored SQL file. + #[derive(Debug)] + pub(crate) struct ModelTextsPy { + pub(crate) name: String, + pub(crate) relative_path: String, + pub(crate) query_sql: String, + pub(crate) authored_query_sql: String, + pub(crate) authored_sql: String, + } +} + +/// Encoded config values and the encoder's error, if any. +pub(crate) type ConfigValues = (BTreeMap, Option); + +pub(crate) fn declaration(row: DeclarationPy<'_>) -> PyResult { + let DeclarationPy(name, relative_path, members, value, value_type, render_as) = row; + Ok(DeclarationFact { + name, + relative_path, + members: members + .into_iter() + .map(|(name, value)| Ok((name, plain_value(&value)?))) + .collect::>()?, + value: Some(plain_value(&value)?).filter(|value| !value.is_null()), + value_type, + render_as, + }) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/schema_facts.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/schema_facts.rs new file mode 100644 index 0000000000..4bb65d4779 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/schema_facts.rs @@ -0,0 +1,43 @@ +//! Native facts of declared schema columns and families, read once from their Python specs. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::record_attribute_opt::record_attribute_opt; + +pub(crate) const NAME_FIELD: &str = "name"; +pub(crate) const TYPE_FIELD: &str = "type"; +pub(crate) const NULLABLE_FIELD: &str = "nullable"; +pub(crate) const DESCRIPTION_FIELD: &str = "description"; +pub(crate) const MIGRATE_FROM_FIELD: &str = "migrate_from"; +pub(crate) const AUDITS_FIELD: &str = "audits"; +pub(crate) const LOCATION_FIELD: &str = "location"; +pub(crate) const PATH_FIELD: &str = "path"; +pub(crate) const DEFINITION_NAME_FIELD: &str = "definition_name"; +pub(crate) const COLUMNS_FIELD: &str = "columns"; +pub(crate) const DYNAMIC_COLUMNS_FIELD: &str = "dynamic_columns"; +pub(crate) const TYPE_ENFORCEMENT_FIELD: &str = "type_enforcement"; +pub(crate) const MODEL_SCHEMA_FIELD: &str = "model_schema"; +pub(crate) const PIVOT_COLUMN_FIELD: &str = "pivot_column"; +pub(crate) const VALUE_COLUMN_FIELD: &str = "value_column"; +pub(crate) const AGGREGATE_FIELD: &str = "aggregate"; +pub(crate) const NAME_PATTERN_FIELD: &str = "name_pattern"; + +/// An attribute's value, `None` when it is missing or `None`. +pub(crate) fn present<'py>( + spec: &Bound<'py, PyAny>, + name: &str, +) -> PyResult>> { + let value: Option> = match record_attribute_opt(spec, name) { + Some(value) => value?, + None => spec.getattr_opt(name)?, + }; + Ok(value.filter(|value| !value.is_none())) +} + +pub(crate) fn optional<'py, T: pyo3::FromPyObject<'py>>( + spec: &Bound<'py, PyAny>, + name: &str, +) -> PyResult> { + present(spec, name)? + .map(|value| value.extract()) + .transpose() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs rename to crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs b/crates/sqlbuild-python-project/src/compiled_project/constants.rs similarity index 59% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs rename to crates/sqlbuild-python-project/src/compiled_project/constants.rs index a265293a21..d72ce96a40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/constants.rs @@ -1,16 +1,26 @@ -//! Frozen execution protocol values and bounded acknowledgement retries. +//! Public constants of compiled project. + +pub const SKIP_ACTION: &str = "skip"; + +pub const NO_CHANGE_REASON: &str = "no_change"; + +pub const REPORT_ATTEMPTS: usize = 3; -pub(crate) const SKIP_ACTION: &str = "skip"; -pub(crate) const NO_CHANGE_REASON: &str = "no_change"; -pub(crate) const REPORT_ATTEMPTS: usize = 3; /// The HTTP status a control server answers while its store is busy; reports retry it. -pub(crate) const STORE_BUSY_STATUS: u16 = 503; +pub const STORE_BUSY_STATUS: u16 = 503; + /// Selector kinds that can name a job: `job:` and a bare name. pub(crate) const JOB_SELECTOR_KINDS: [&str; 2] = ["job", "name"]; + /// Separates the parts of an intersection selector. pub(crate) const SELECTOR_INTERSECTION: char = ','; + /// The key prefix of a SQL unit test's result, as the snapshot names it. -pub(crate) const SQL_TEST_KEY_PREFIX: &str = "sql_test:"; +pub const SQL_TEST_KEY_PREFIX: &str = "sql_test:"; /// The sensor argument naming the job it starts, as the orchestration inventory labels it. pub(crate) const SENSOR_RUN_ARGUMENT: &str = "run"; + +pub const CONTRACT_KEY: &str = "contract"; + +pub const INCREMENTAL_MODE_KEY: &str = "incremental_mode"; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/config_values.rs b/crates/sqlbuild-python-project/src/compiled_project/main/config_values.rs new file mode 100644 index 0000000000..7ea19c4621 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/config_values.rs @@ -0,0 +1,17 @@ +//! The config values, or the encoder's error, kept for the consumer that reads them. + +use crate::compiled_project::_helpers::project::ConfigValues; +use crate::compiled_project::_helpers::values::object_value; +use pyo3::prelude::Bound; +use pyo3::types::PyDict; +use serde_json::Value; +use std::collections::BTreeMap; + +/// The config values, or the encoder's error, kept for the consumer that reads them. +pub fn config_values(values: &Bound<'_, PyDict>) -> ConfigValues { + match object_value(values) { + Ok(Value::Object(entries)) => (entries.into_iter().collect(), None), + Ok(_) => (BTreeMap::new(), None), + Err(error) => (BTreeMap::new(), Some(error.to_string())), + } +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/declarations.rs b/crates/sqlbuild-python-project/src/compiled_project/main/declarations.rs new file mode 100644 index 0000000000..9e50d50c3a --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/declarations.rs @@ -0,0 +1,11 @@ +//! Declarations. + +use crate::compiled_project::_helpers::project::declaration; +use crate::compiled_project::models::DeclarationPy; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::compiled_project::models::DeclarationFact; + +/// Declarations. +pub fn declarations(rows: Vec>) -> PyResult> { + rows.into_iter().map(declaration).collect() +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/entry_schema_fact.rs b/crates/sqlbuild-python-project/src/compiled_project/main/entry_schema_fact.rs new file mode 100644 index 0000000000..6be3f0d8e0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/entry_schema_fact.rs @@ -0,0 +1,48 @@ +//! The fact of a hand-built model's `SchemaModelEntry`, or `None` without one. + +use crate::compiled_project::_helpers::schema_facts::AGGREGATE_FIELD; +use crate::compiled_project::_helpers::schema_facts::AUDITS_FIELD; +use crate::compiled_project::_helpers::schema_facts::COLUMNS_FIELD; +use crate::compiled_project::_helpers::schema_facts::DYNAMIC_COLUMNS_FIELD; +use crate::compiled_project::_helpers::schema_facts::MODEL_SCHEMA_FIELD; +use crate::compiled_project::_helpers::schema_facts::NAME_FIELD; +use crate::compiled_project::_helpers::schema_facts::NAME_PATTERN_FIELD; +use crate::compiled_project::_helpers::schema_facts::PIVOT_COLUMN_FIELD; +use crate::compiled_project::_helpers::schema_facts::TYPE_ENFORCEMENT_FIELD; +use crate::compiled_project::_helpers::schema_facts::TYPE_FIELD; +use crate::compiled_project::_helpers::schema_facts::VALUE_COLUMN_FIELD; +use crate::compiled_project::main::family_fact::family_fact; +use crate::compiled_project::main::python_column_facts::python_column_facts; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::compiled_project::models::{DynamicFamilyFact, ModelSchemaFact}; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The fact of a hand-built model's `SchemaModelEntry`, or `None` without one. +pub fn entry_schema_fact( + entry: &Bound<'_, PyAny>, + model_path: &Bound<'_, PyAny>, +) -> PyResult> { + if entry.is_none() { + return Ok(None); + } + let mut dynamic_columns: Vec = Vec::new(); + for family in native_field(entry, DYNAMIC_COLUMNS_FIELD)?.try_iter()? { + let family: Bound<'_, PyAny> = family?; + dynamic_columns.push(family_fact(&DynamicFamily { + name: native_field(&family, NAME_FIELD)?.extract()?, + pivot_column: native_field(&family, PIVOT_COLUMN_FIELD)?.extract()?, + value_column: native_field(&family, VALUE_COLUMN_FIELD)?.extract()?, + aggregate: native_field(&family, AGGREGATE_FIELD)?.extract()?, + data_type: native_field(&family, TYPE_FIELD)?.extract()?, + name_pattern: native_field(&family, NAME_PATTERN_FIELD)?.extract()?, + })); + } + Ok(Some(ModelSchemaFact { + audit_count: u32::try_from(native_field(entry, AUDITS_FIELD)?.len()?).unwrap_or(u32::MAX), + columns: python_column_facts(&native_field(entry, COLUMNS_FIELD)?, Some(model_path))?, + dynamic_columns, + type_enforcement: native_field(entry, TYPE_ENFORCEMENT_FIELD)?.is_truthy()?, + named_schema: !native_field(entry, MODEL_SCHEMA_FIELD)?.is_none(), + })) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/family_fact.rs b/crates/sqlbuild-python-project/src/compiled_project/main/family_fact.rs new file mode 100644 index 0000000000..532f7edda9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/family_fact.rs @@ -0,0 +1,18 @@ +//! A parsed dynamic column family's fact, keyed by its casefolded name. + +use sqlbuild_analysis::compiled_project::models::DynamicFamilyFact; +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; + +/// A parsed dynamic column family's fact, keyed by its casefolded name. +pub fn family_fact(family: &DynamicFamily) -> DynamicFamilyFact { + DynamicFamilyFact { + key: python_casefold(&family.name), + name: family.name.clone(), + pivot_column: family.pivot_column.clone(), + value_column: family.value_column.clone(), + aggregate: family.aggregate.clone(), + data_type: family.data_type.clone(), + name_pattern: family.name_pattern.clone(), + } +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/graph_with_jobs.rs b/crates/sqlbuild-python-project/src/compiled_project/main/graph_with_jobs.rs new file mode 100644 index 0000000000..e8fda5ea27 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/graph_with_jobs.rs @@ -0,0 +1,20 @@ +//! `graph` with the command's jobs attached when `selectors` name a job, otherwise `graph` itself. + +use crate::compiled_project::main::needed_jobs::needed_jobs; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; +use sqlbuild_analysis::graph::models::ProjectGraph; +use std::borrow::Cow; + +/// `graph` with the command's jobs attached when `selectors` name a job, otherwise `graph` itself. +pub fn graph_with_jobs<'g>( + py: Python<'_>, + graph: &'g ProjectGraph, + selectors: &[String], + also_known: &[&str], +) -> PyResult> { + Ok(match needed_jobs(py, graph, selectors, also_known)? { + Some(catalog) => Cow::Owned(attach_jobs(graph.clone(), catalog)), + None => Cow::Borrowed(graph), + }) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/install.rs b/crates/sqlbuild-python-project/src/compiled_project/main/install.rs new file mode 100644 index 0000000000..ee2e1117e2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/install.rs @@ -0,0 +1,9 @@ +//! Install the compile's attachment runs once, at registration. + +use crate::compiled_project::_helpers::attachment_inputs::INSTALLED; +use crate::compiled_project::models::ResourceAttachment; + +/// Install the compile's attachment runs once, at registration. +pub fn install(attachment: ResourceAttachment) { + let _ = INSTALLED.get_or_init(|| attachment); +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_names.rs b/crates/sqlbuild-python-project/src/compiled_project/main/job_names.rs new file mode 100644 index 0000000000..bd3875b222 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/job_names.rs @@ -0,0 +1,11 @@ +//! The command's job and job group names, loaded on first use; empty outside a command. + +use crate::compiled_project::_helpers::job_catalog::job_catalog; +use pyo3::prelude::{PyResult, Python}; + +/// The command's job and job group names, loaded on first use; empty outside a command. +pub fn job_names(py: Python<'_>) -> PyResult> { + Ok(job_catalog(py)? + .map(|(catalog, _)| catalog.into_keys().collect()) + .unwrap_or_default()) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/materialization.rs b/crates/sqlbuild-python-project/src/compiled_project/main/materialization.rs new file mode 100644 index 0000000000..ca4573eee6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/materialization.rs @@ -0,0 +1,15 @@ +//! The `materialized` config value when it is text, `table` when it is absent. + +use crate::compiled_project::_helpers::project::MATERIALIZED_KEY; +use crate::compiled_project::_helpers::project::TABLE_MATERIALIZATION; +use crate::compiled_project::main::text_value::text_value; +use pyo3::prelude::{Bound, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; + +/// The `materialized` config value when it is text, `table` when it is absent. +pub fn materialization(values: &Bound<'_, PyDict>) -> PyResult> { + if values.get_item(MATERIALIZED_KEY)?.is_none() { + return Ok(Some(TABLE_MATERIALIZATION.to_owned())); + } + text_value(values, MATERIALIZED_KEY) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs new file mode 100644 index 0000000000..1a34adb72d --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs @@ -0,0 +1,18 @@ +pub mod config_values; +pub mod declarations; +pub mod entry_schema_fact; +pub mod family_fact; +pub mod graph_with_jobs; +pub mod install; +pub mod job_names; +pub mod materialization; +pub mod model_facts; +pub mod needed_jobs; +pub mod plain_value; +pub mod python_column_fact; +pub mod python_column_facts; +pub mod reference_row_tuple; +pub mod register_project; +pub mod text_value; +pub mod triggered_jobs; +pub mod with_job_scope; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/model_facts.rs b/crates/sqlbuild-python-project/src/compiled_project/main/model_facts.rs new file mode 100644 index 0000000000..ce21e47ad3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/model_facts.rs @@ -0,0 +1,65 @@ +//! Model facts. + +use crate::compiled_project::_helpers::project::ConfigValues; +use crate::compiled_project::_helpers::project::ModelTextsPy; +use crate::compiled_project::constants::CONTRACT_KEY; +use crate::compiled_project::constants::INCREMENTAL_MODE_KEY; +use crate::compiled_project::main::config_values::config_values; +use crate::compiled_project::main::declarations::declarations; +use crate::compiled_project::main::materialization::materialization; +use crate::compiled_project::main::text_value::text_value; +use crate::compiled_project::models::ModelInputPy; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::compiled_project::models::{ + CompiledModelFacts, ModelConfigFacts, ModelReferenceFact, +}; + +/// Model facts. +pub fn model_facts(model: ModelInputPy<'_>) -> PyResult { + let ModelInputPy( + ModelTextsPy { + name, + relative_path, + query_sql, + authored_query_sql, + authored_sql, + }, + (values, header_keys, layer_schema), + references, + enum_columns, + (enum_declarations, constant_declarations), + ) = model; + let config_values: ConfigValues = config_values(&values); + Ok(CompiledModelFacts { + name, + relative_path, + query_sql: query_sql.into(), + authored_query_sql: authored_query_sql.into(), + authored_sql: authored_sql.into(), + config: ModelConfigFacts { + values: config_values.0, + encoding_error: config_values.1, + header_keys, + layer_schema, + contract: text_value(&values, CONTRACT_KEY)?, + materialized: materialization(&values)?, + incremental_mode: text_value(&values, INCREMENTAL_MODE_KEY)?, + }, + references: references + .into_iter() + .map(|(kind, name, package)| ModelReferenceFact { + kind, + name, + package, + }) + .collect(), + schema: None, + enum_columns, + enum_declarations: declarations(enum_declarations)?, + constant_declarations: declarations(constant_declarations)?, + analysis: None, + typed_calls: Vec::new(), + expansion_passes: Vec::new(), + sql_analysis_skipped: false, + }) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/needed_jobs.rs b/crates/sqlbuild-python-project/src/compiled_project/main/needed_jobs.rs new file mode 100644 index 0000000000..949d525f66 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/needed_jobs.rs @@ -0,0 +1,20 @@ +//! The command's jobs when `selectors` name one and `graph` has none attached yet. + +use crate::compiled_project::_helpers::job_catalog::job_catalog; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; +use sqlbuild_analysis::graph::models::{JobSelection, ProjectGraph}; +use std::collections::BTreeMap; + +/// The command's jobs when `selectors` name one and `graph` has none attached yet. +pub fn needed_jobs( + py: Python<'_>, + graph: &ProjectGraph, + selectors: &[String], + also_known: &[&str], +) -> PyResult>> { + if graph.jobs.is_some() || !selectors_need_jobs(graph, selectors, also_known) { + return Ok(None); + } + Ok(job_catalog(py)?.map(|(catalog, _)| catalog)) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/plain_value.rs b/crates/sqlbuild-python-project/src/compiled_project/main/plain_value.rs new file mode 100644 index 0000000000..3103672450 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/plain_value.rs @@ -0,0 +1,11 @@ +//! orjson's value for `value`: enums by value, dataclasses by public attribute, dates in ISO form. + +use crate::compiled_project::_helpers::values; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use serde_json::Value; + +/// orjson's value for `value`: enums by value, dataclasses by public attribute, dates in ISO form. +#[inline] +pub fn plain_value(value: &Bound<'_, PyAny>) -> PyResult { + values::plain_value(value) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/python_column_fact.rs b/crates/sqlbuild-python-project/src/compiled_project/main/python_column_fact.rs new file mode 100644 index 0000000000..ad733482c6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/python_column_fact.rs @@ -0,0 +1,51 @@ +//! A Python column spec's fact; its location is kept when it differs from `model_path`. + +use crate::compiled_project::_helpers::schema_facts::AUDITS_FIELD; +use crate::compiled_project::_helpers::schema_facts::DEFINITION_NAME_FIELD; +use crate::compiled_project::_helpers::schema_facts::DESCRIPTION_FIELD; +use crate::compiled_project::_helpers::schema_facts::LOCATION_FIELD; +use crate::compiled_project::_helpers::schema_facts::MIGRATE_FROM_FIELD; +use crate::compiled_project::_helpers::schema_facts::NAME_FIELD; +use crate::compiled_project::_helpers::schema_facts::NULLABLE_FIELD; +use crate::compiled_project::_helpers::schema_facts::PATH_FIELD; +use crate::compiled_project::_helpers::schema_facts::TYPE_FIELD; +use crate::compiled_project::_helpers::schema_facts::optional; +use crate::compiled_project::_helpers::schema_facts::present; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A Python column spec's fact; its location is kept when it differs from `model_path`. +pub fn python_column_fact( + column: &Bound<'_, PyAny>, + model_path: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let mut audit_definitions: Vec = Vec::new(); + if let Some(audits) = present(column, AUDITS_FIELD)? { + for audit in audits.try_iter()? { + audit_definitions.push(native_field(&audit?, DEFINITION_NAME_FIELD)?.extract()?); + } + } + let location_path: Option = match present(column, LOCATION_FIELD)? { + Some(location) => { + let path: Bound<'_, PyAny> = native_field(&location, PATH_FIELD)?; + let outside: bool = match model_path { + Some(model_path) => !path.is(model_path) && path.ne(model_path)?, + None => true, + }; + outside + .then(|| path.str().map(|text| text.to_string())) + .transpose()? + } + None => None, + }; + Ok(SchemaColumnFact { + name: native_field(column, NAME_FIELD)?.str()?.extract()?, + data_type: optional(column, TYPE_FIELD)?, + nullable: optional(column, NULLABLE_FIELD)?, + described: present(column, DESCRIPTION_FIELD)?.is_some(), + migrate_from: optional(column, MIGRATE_FROM_FIELD)?, + audit_definitions, + location_path, + }) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/python_column_facts.rs b/crates/sqlbuild-python-project/src/compiled_project/main/python_column_facts.rs new file mode 100644 index 0000000000..d0096876f3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/python_column_facts.rs @@ -0,0 +1,16 @@ +//! The facts of every column spec in `columns`. + +use crate::compiled_project::main::python_column_fact::python_column_fact; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; + +/// The facts of every column spec in `columns`. +pub fn python_column_facts( + columns: &Bound<'_, PyAny>, + model_path: Option<&Bound<'_, PyAny>>, +) -> PyResult> { + columns + .try_iter()? + .map(|column| python_column_fact(&column?, model_path)) + .collect() +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/reference_row_tuple.rs b/crates/sqlbuild-python-project/src/compiled_project/main/reference_row_tuple.rs new file mode 100644 index 0000000000..252124f7ec --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/reference_row_tuple.rs @@ -0,0 +1,20 @@ +//! A reference as the tuple Python reads. + +use crate::compiled_project::models::ReferenceTuple; +use sqlbuild_project::models::ReferenceRow; + +/// A reference as the tuple Python reads. +pub fn reference_row_tuple(row: ReferenceRow) -> ReferenceTuple { + let ReferenceRow { + kind, + name, + package, + call_argument_count, + } = row; + ReferenceTuple { + kind, + name, + package, + call_argument_count, + } +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/register_project.rs b/crates/sqlbuild-python-project/src/compiled_project/main/register_project.rs new file mode 100644 index 0000000000..033b3ff829 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/register_project.rs @@ -0,0 +1,29 @@ +//! Register the project functions and classes. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeInputsView; +use crate::compiled_views::models::NativeModelInput; +use crate::compiled_views::models::NativeProjectView; +use crate::resource_views::models::{NativeAuditInput, NativeCompiledAudit}; +use crate::resource_views::models::{NativeCompiledFunction, NativeFunctionInput}; +use crate::resource_views::models::{NativeCompiledSeed, NativeSeedInput}; +use crate::resource_views::models::{NativeCompiledSource, NativeSourceInput}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the project functions and classes. +pub fn register_project(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::() +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs b/crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs new file mode 100644 index 0000000000..6003013b76 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs @@ -0,0 +1,12 @@ +//! A config value when it is text. + +use pyo3::prelude::{Bound, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; + +/// A config value when it is text. +pub fn text_value(values: &Bound<'_, PyDict>, key: &str) -> PyResult> { + match values.get_item(key)? { + Some(value) if value.is_instance_of::() => Ok(Some(value.extract()?)), + _ => Ok(None), + } +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/triggered_jobs.rs b/crates/sqlbuild-python-project/src/compiled_project/main/triggered_jobs.rs new file mode 100644 index 0000000000..741f09e695 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/triggered_jobs.rs @@ -0,0 +1,40 @@ +//! Each job a schedule or sensor starts with the nodes it refreshes, less those of jobs `selectors` run. + +use crate::compiled_project::_helpers::job_catalog::job_catalog; +use crate::compiled_project::_helpers::job_catalog::selected_jobs; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; +use sqlbuild_analysis::graph::models::ProjectGraph; +use std::collections::BTreeSet; + +/// Each job a schedule or sensor starts with the nodes it refreshes, less those of jobs `selectors` run. +pub fn triggered_jobs( + py: Python<'_>, + graph: &ProjectGraph, + selectors: &[String], +) -> PyResult)>> { + let Some((catalog, triggered)) = job_catalog(py)? else { + return Ok(Vec::new()); + }; + if triggered.is_empty() { + return Ok(Vec::new()); + } + let running: BTreeSet = selected_jobs(selectors, &catalog); + let mut graph: ProjectGraph = attach_jobs(graph.clone(), catalog); + let mut resolved = graph.jobs.take().unwrap_or_default(); + let mut running_nodes: BTreeSet = BTreeSet::new(); + for name in &running { + if let Some(Ok(keys)) = resolved.get(name) { + running_nodes.extend(keys.iter().cloned()); + } + } + Ok(triggered + .into_iter() + .filter(|name| !running.contains(name)) + .filter_map(|name| match resolved.remove(&name) { + Some(Ok(keys)) => Some((name, &keys - &running_nodes)), + _ => None, + }) + .collect()) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/with_job_scope.rs b/crates/sqlbuild-python-project/src/compiled_project/main/with_job_scope.rs new file mode 100644 index 0000000000..bc8fde79ad --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/with_job_scope.rs @@ -0,0 +1,13 @@ +//! Run one command with jobs selectable from `project_dir`, restoring the enclosing scope after. + +use crate::compiled_project::_helpers::job_catalog; +use std::path::PathBuf; + +/// Run one command with jobs selectable from `project_dir`, restoring the enclosing scope after. +#[inline] +pub fn with_job_scope( + (project_dir, selected_target): (PathBuf, Option), + run: impl FnOnce() -> T, +) -> T { + job_catalog::with_job_scope((project_dir, selected_target), run) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/mod.rs new file mode 100644 index 0000000000..4af01a464c --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/mod.rs @@ -0,0 +1,7 @@ +//! The compiled project: its rows, attachment inputs, schema facts, values and job catalog. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/compiled_project/models.rs b/crates/sqlbuild-python-project/src/compiled_project/models.rs new file mode 100644 index 0000000000..73eb20c543 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/models.rs @@ -0,0 +1,1025 @@ +//! Public records of compiled project. + +use crate::compiled_project::_helpers::attachment_inputs::AttachAudits; +use crate::compiled_project::_helpers::attachment_inputs::AttachFunctions; +use crate::compiled_project::_helpers::attachment_inputs::AttachSources; +use crate::compiled_project::_helpers::attachment_inputs::AttachSqlTests; +use crate::compiled_project::_helpers::attachment_inputs::ResourceDiagnosticsRun; +use crate::compiled_project::_helpers::attachment_inputs::resource_attachment; +use crate::compiled_project::_helpers::project::ATTACHMENT_LOOP_UNSET; +use crate::compiled_project::_helpers::project::INPUT_CLASSES_UNSET; +use crate::compiled_project::_helpers::project::ModelTextsPy; +use crate::compiled_project::_helpers::project::NamedType; +use crate::compiled_project::types::AttachedSqlTests; +use crate::compiled_project::types::FunctionSettingsTuple; +use crate::compiled_project::types::LoaderRow; +use crate::compiled_project::types::ResourceDiagnostics; +use crate::compiled_project::types::SourceRow; +use crate::compiled_project::types::SourceSettings; +use crate::compiled_project::types::TestBlockTuple; +use crate::compiled_project::types::TestCaseRow; +use crate::compiled_views::models::CompiledViewClasses; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeInputsView; +use crate::compiled_views::models::NativeProjectView; +use crate::compiled_views::models::PyProjectHost; +use crate::compiled_views::types::ModelInputRecord; +use crate::resource_views::main::audit_views::audit_record::audit_record; +use crate::resource_views::main::function_views::function_record::function_record; +use crate::resource_views::main::seed_views::paired_seeds::paired_seeds; +use crate::resource_views::models::NativeAuditInput; +use crate::resource_views::models::NativeCompiledAudit; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; +use crate::resource_views::models::NativeFunctionInput; +use crate::resource_views::models::NativeSeedInput; +use crate::resource_views::models::NativeSourceInput; +use crate::resource_views::models::SeedNamespace; +use crate::resource_views::types::AttachedAudit; +use crate::resource_views::types::AttachedFunction; +use crate::resource_views::types::AuditRecord; +use crate::resource_views::types::FunctionRecord; +use crate::resource_views::types::SeedRecord; +use crate::resource_views::types::SeedViews; +use crate::resource_views::types::SourceRecord; +use pyo3::FromPyObject; +use pyo3::exceptions::PyIndexError; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyRef; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyString; +use sqlbuild_analysis::compiled_project::main::project_models::project_models; +use sqlbuild_analysis::compiled_project::main::record_model_analysis::record_model_analysis; +use sqlbuild_analysis::compiled_project::main::record_skipped_analysis::record_skipped_analysis; +use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; +use sqlbuild_analysis::compiled_project::models::CompiledProjectFacts; +use sqlbuild_analysis::compiled_project::models::DynamicProofFact; +use sqlbuild_analysis::compiled_project::models::ModelAnalysisFact; +use sqlbuild_analysis::compiled_project::models::ModelSchemaFact; +use sqlbuild_project::models::CompiledProject; +use sqlbuild_project::types::InferredRow; +use sqlbuild_project::types::ObjectKeyRow; +use sqlbuild_project::types::SourceNamespace; +use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; +use sqlbuild_python_base::cache::models::SqlTestScanStore; +use sqlbuild_python_front::render::models::ConstantRendering; +use std::collections::HashMap; +use std::sync::Mutex; + +/// A function's paths, stem, discovered file, header values, body and SQL file contents. +#[derive(Debug)] +pub struct FunctionRow { + pub file_path: String, + pub relative_path: String, + pub name: String, + pub function_file: Py, + pub header_values: Py, + pub body: String, + pub contents: Option, +} + +impl<'py> pyo3::FromPyObject<'py> for FunctionRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (file_path, relative_path, name, function_file, header_values, body, contents): ( + String, + String, + String, + Py, + Py, + String, + Option, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + file_path, + relative_path, + name, + function_file, + header_values, + body, + contents, + }) + } +} + +/// Adapter, validation, raw default and target namespaces, inheritance and resource kind. +#[derive(Debug)] +pub struct FunctionSettings { + pub adapter_name: String, + pub validate: bool, + pub defaults: (Option, Option), + pub target: Option<(Option, Option)>, + pub inherit: bool, + pub kind: Py, +} + +impl<'py> pyo3::FromPyObject<'py> for FunctionSettings { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (adapter_name, validate, defaults, target, inherit, kind): FunctionSettingsTuple = + pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + adapter_name, + validate, + defaults, + target, + inherit, + kind, + }) + } +} + +/// A test block, its name, SQL, mode, declared parameter names, cases and authored SQL span. +#[derive(Debug)] +pub struct TestBlockRow { + pub block: Py, + pub name: Option, + pub sql_body: String, + pub mode: String, + pub parameters: Vec, + pub cases: Vec, + pub sql_body_span: Option<(usize, usize)>, +} + +impl<'py> pyo3::FromPyObject<'py> for TestBlockRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (block, name, sql_body, mode, parameters, cases, sql_body_span): TestBlockTuple = + pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + block, + name, + sql_body, + mode, + parameters, + cases, + sql_body_span, + }) + } +} + +/// A test file's paths, stem, discovered file, contents and blocks. +#[derive(Debug)] +pub struct TestFileRow { + pub file_path: String, + pub relative_path: String, + pub stem: String, + pub file: Py, + pub contents: String, + pub blocks: Vec, +} + +impl<'py> pyo3::FromPyObject<'py> for TestFileRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (file_path, relative_path, stem, file, contents, blocks): ( + String, + String, + String, + Py, + String, + Vec, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + file_path, + relative_path, + stem, + file, + contents, + blocks, + }) + } +} + +/// What every SQL test and scenario of one compile resolves and records with. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub struct SqlTestSettings { + /// `CompileSqlTestInput`, its model and direct payloads, and `CompileSqlTestCte`. + pub test_input: Py, + pub model_payload: Py, + pub direct_payload: Py, + pub test_cte: Py, + /// `CompileSqlScenarioInput` and `CompileSqlScenarioCte`. + pub scenario_input: Py, + pub scenario_cte: Py, + /// `SqlTestMode` by value. + pub modes: Py, + /// The `ResourceKind`s of tests and scenarios, and the runtime `UsageKind`. + pub test_kind: Py, + pub scenario_kind: Py, + pub runtime_usage: Py, + /// The adapter's typed scalar renderer, returning `(text, None)` or `(None, error)`. + pub render_parameter: Py, + pub adapter_name: String, + /// `SourceLocation`, for helper problems located natively. + pub source_location: Py, + /// The shared scan store with the scan algorithm and lexical rules keying it, if any. + pub scan_store: Option>, + pub scan_key: (String, String), +} + +/// A scenario file's paths, discovered file, contents, SQL body and name. +#[derive(Debug)] +pub struct ScenarioRow { + pub file_path: String, + pub relative_path: String, + pub scenario_file: Py, + pub contents: String, + pub sql_body: String, + pub name: String, +} + +impl<'py> pyo3::FromPyObject<'py> for ScenarioRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (file_path, relative_path, scenario_file, contents, sql_body, name): ( + String, + String, + Py, + String, + String, + String, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + file_path, + relative_path, + scenario_file, + contents, + sql_body, + name, + }) + } +} + +/// An audit block, its name, SQL body, evidence SQL, measurement flag and header values. +#[derive(Debug)] +pub struct AuditBlockRow { + pub block: Py, + pub name: Option, + pub sql_body: String, + pub evidence_sql: Option, + pub measurement: bool, + pub header: Py, +} + +impl<'py> pyo3::FromPyObject<'py> for AuditBlockRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (block, name, sql_body, evidence_sql, measurement, header): ( + Py, + Option, + String, + Option, + bool, + Py, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + block, + name, + sql_body, + evidence_sql, + measurement, + header, + }) + } +} + +impl<'py> pyo3::IntoPyObject<'py> for AuditBlockRow { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject( + ( + self.block, + self.name, + self.sql_body, + self.evidence_sql, + self.measurement, + self.header, + ), + py, + ) + } +} + +/// An audit file's paths, stem, discovered file, generic flag, contents and blocks. +#[derive(Debug)] +pub struct AuditFileRow { + pub file_path: String, + pub relative_path: String, + pub stem: String, + pub file: Py, + pub generic: bool, + pub contents: String, + pub blocks: Vec, +} + +impl<'py> pyo3::FromPyObject<'py> for AuditFileRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (file_path, relative_path, stem, file, generic, contents, blocks): ( + String, + String, + String, + Py, + bool, + String, + Vec, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + file_path, + relative_path, + stem, + file, + generic, + contents, + blocks, + }) + } +} + +impl<'py> pyo3::IntoPyObject<'py> for AuditFileRow { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject( + ( + self.file_path, + self.relative_path, + self.stem, + self.file, + self.generic, + self.contents, + self.blocks, + ), + py, + ) + } +} + +/// What every audit in one compile resolves against, and the Python values it builds with. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub struct AuditSettings { + pub default_severity: Option, + pub default_run_scope: Option, + /// Built-in generic definitions as audit file rows, overridden by project definitions. + pub builtins: Vec, + /// `DeclarationIdentity`, and the singular and generic audit `DeclarationKind`s. + pub declaration_identity: Py, + pub singular_kind: Py, + pub audit_kind: Py, + /// `ResourceKind` by value, for the resources singular audits read and attach to. + pub resource_kinds: Py, +} + +/// A Python resource's kind, name, description (a `str` that may hold surrogates), location, help. +#[derive(Debug)] +pub struct PythonResourceRow { + pub kind: String, + pub name: String, + pub description: Option>, + pub location: Py, + pub help: String, +} + +impl<'py> pyo3::FromPyObject<'py> for PythonResourceRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (kind, name, description, location, help): ( + String, + String, + Option>, + Py, + String, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + kind, + name, + description, + location, + help, + }) + } +} + +/// A scenario's name, relative path, contents and authored header description. +#[derive(Debug)] +pub struct ScenarioDescriptionRow { + pub name: String, + pub path: String, + pub contents: String, + pub raw: Py, +} + +impl<'py> pyo3::FromPyObject<'py> for ScenarioDescriptionRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (name, path, contents, raw): (String, String, String, Py) = + pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + name, + path, + contents, + raw, + }) + } +} + +/// A SQL hook file's name, relative path, contents and description. +#[derive(Debug)] +pub struct HookDescriptionRow { + pub name: String, + pub path: String, + pub contents: String, + pub description: Option, +} + +impl<'py> pyo3::FromPyObject<'py> for HookDescriptionRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (name, path, contents, description): (String, String, String, Option) = + pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + name, + path, + contents, + description, + }) + } +} + +/// What the resource diagnostics read beyond the owner, and the Python values they build. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub struct DiagnosticSettings { + /// `CompilerDiagnostic`, `SourceLocation`, `Path` and `CompileInputError`. + pub compiler_diagnostic: Py, + pub compile_error: Py, + pub source_location: Py, + pub path_class: Py, + /// Every discovered model file, whose headers attach audit factories. + pub model_files: Vec>, + /// Each discovered audit factory's name, relative path and line. + pub audit_factories: Vec<(String, String, usize)>, + /// Each scenario's name, relative path, contents and authored header description. + pub scenarios: Vec, + /// Each SQL hook file's name, relative path, contents and description. + pub sql_hooks: Vec, + /// Each loader by name. + pub loaders: HashMap, + /// Python-defined resources: kind, name, description, location and help, read by Python. + pub python_resources: Vec, + /// The project's `max_description_lines` and `line_width`, which Python descriptions keep to. + pub description_limit: (usize, i64), +} + +/// The compile's runs behind the project's attachment methods. +#[derive(Clone, Copy, Debug)] +pub struct ResourceAttachment { + pub sources: AttachSources, + pub functions: AttachFunctions, + pub diagnostics: ResourceDiagnosticsRun, + pub sql_tests: AttachSqlTests, + pub audits: AttachAudits, +} + +/// `(output proven, bare dynamic pivot, (family key, inferred type), failure, (family, type))`. +#[derive(Debug)] +pub struct ProofPy { + pub output_proven: bool, + pub bare_dynamic_pivot: bool, + pub families: Vec, + pub failure_reason: Option, + pub named_families: Vec, +} + +impl<'py> pyo3::FromPyObject<'py> for ProofPy { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (output_proven, bare_dynamic_pivot, families, failure_reason, named_families): ( + bool, + bool, + Vec, + Option, + Vec, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + output_proven, + bare_dynamic_pivot, + families, + failure_reason, + named_families, + }) + } +} + +/// `(inferred, (name, type), nullable per column, has star, unchecked output columns)`. +#[derive(Debug)] +pub struct AnalysisPy { + pub inferred: bool, + pub inferred_columns: Vec, + pub nullable_columns: Vec, + pub fast_lineage_has_star: bool, + pub unchecked: Vec, +} + +impl<'py> pyo3::FromPyObject<'py> for AnalysisPy { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (inferred, inferred_columns, nullable_columns, fast_lineage_has_star, unchecked): ( + bool, + Vec, + Vec, + bool, + Vec, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + inferred, + inferred_columns, + nullable_columns, + fast_lineage_has_star, + unchecked, + }) + } +} + +/// One model as the compile-input stage produced it. +#[derive(Debug, FromPyObject)] +pub struct ModelInputPy<'py>( + pub(crate) ModelTextsPy, + pub(crate) (Bound<'py, PyDict>, Vec, Option), + pub(crate) Vec<(String, String, Option)>, + pub(crate) Vec, + pub(crate) (Vec>, Vec>), +); + +/// Name, relative path, members, value, value type and rendering of one declaration. +#[derive(Debug, FromPyObject)] +pub struct DeclarationPy<'py>( + pub(crate) String, + pub(crate) String, + pub(crate) Vec<(String, Bound<'py, PyAny>)>, + pub(crate) Bound<'py, PyAny>, + pub(crate) Option, + pub(crate) Option, +); + +/// One compile's project, recorded by the stages that produce it, with its Python projections. +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug, Default)] +pub struct NativeCompiledProject { + pub(crate) project: CompiledProject, + /// The Python classes model input views project their attributes with. + pub(crate) input_classes: Option>, + /// Shared immutable column projections, allocated only when Python consumers request them. + pub(crate) inferred_column_projections: Mutex>>, + /// The model loop and its constant rendering, which attach the project's other resources. + pub(crate) attachment: Option<(Py, Py)>, +} + +impl NativeCompiledProject { + /// The facts recorded so far, for native consumers such as the rules request. + pub fn facts(&self) -> &CompiledProjectFacts { + &self.project.facts + } + + /// A project holding only `facts`, which analyses can be recorded into apart from the compile's. + pub fn with_facts(facts: CompiledProjectFacts) -> Self { + Self { + project: CompiledProject::with_facts(facts), + ..Self::default() + } + } + + /// Retain facts a producer finished natively. + pub fn retain(&mut self, facts: CompiledModelFacts) { + self.project.retain(facts); + } + + /// Retain a finished model's input record; return its index for the model's view. + pub fn retain_input(&mut self, record: ModelInputRecord) -> usize { + self.project.retain_input(record) + } + + /// Keep the classes model input views project their attributes with. + pub fn set_input_classes(&mut self, classes: Py) { + self.input_classes = Some(classes); + } + + /// Count one lineage graph a project view built. + pub fn record_graph_build(&mut self) { + self.project.graph_builds += 1; + } + + /// The kept model loop and its constant rendering. + pub fn attachment_loop(&self, py: Python<'_>) -> PyResult<(Py, Py)> { + match &self.attachment { + Some((model_loop, host)) => Ok((model_loop.clone_ref(py), host.clone_ref(py))), + None => Err(PyRuntimeError::new_err(ATTACHMENT_LOOP_UNSET)), + } + } + + /// Keep the model loop and its constant rendering for attaching the project's other resources. + pub fn keep_attachment_loop(&mut self, model_loop: Py, host: Py) { + self.attachment = Some((model_loop, host)); + } + + /// Retain attached functions; return the index of the first. + pub fn extend_functions(&mut self, attached: Vec) -> usize { + let first: usize = self.project.functions.len(); + self.project + .functions + .extend(attached.into_iter().map(function_record)); + first + } + + /// Retain attached audits; return the index of the first. + pub fn extend_audits(&mut self, attached: Vec) -> usize { + let first: usize = self.project.audits.len(); + self.project + .audits + .extend(attached.into_iter().map(audit_record)); + first + } + + /// Every attached audit, in attachment order. + pub fn audits(&self) -> &[AuditRecord] { + self.project.audits() + } + + /// Every attached function, in attachment order. + pub fn functions(&self) -> &[FunctionRecord] { + self.project.functions() + } + + /// Every attached source, in attachment order. + pub fn sources(&self) -> &[SourceRecord] { + self.project.sources() + } + + /// Keep one attached source. + pub fn retain_source(&mut self, source: SourceRecord) { + self.project.sources.push(source); + } + + /// Every attached seed, in attachment order. + pub fn seeds(&self) -> &[SeedRecord] { + self.project.seeds() + } + + /// The schema fact recorded for an input record's model. + pub fn input_schema_fact(&self, record: &ModelInputRecord) -> Option<&ModelSchemaFact> { + self.project.input_schema_fact(record) + } + + /// Every model's input record, in loop order. + pub fn inputs(&self) -> &[ModelInputRecord] { + &self.project.inputs + } + + /// The input record at `index`. + pub fn input(&self, index: usize) -> PyResult<&ModelInputRecord> { + self.project + .input(index) + .ok_or_else(|| PyIndexError::new_err(index)) + } + + /// The input record at `index`, for recording what later stages finish. + pub fn input_mut(&mut self, index: usize) -> PyResult<&mut ModelInputRecord> { + self.project + .inputs + .get_mut(index) + .ok_or_else(|| PyIndexError::new_err(index)) + } + + /// Record that the native parser skipped the model at `index`: it is not analysed. + pub fn mark_analysis_skipped(&mut self, index: usize) -> PyResult<()> { + let record: &mut ModelInputRecord = self.input_mut(index)?; + record.sql_validation_enabled = false; + let name: String = record.name.clone(); + record_skipped_analysis(&mut self.project.facts, &name); + Ok(()) + } + + /// The classes model input views project their attributes with. + pub fn input_classes<'py>(&self, py: Python<'py>) -> PyResult> { + self.input_classes + .as_ref() + .map(|classes| classes.bind(py).clone()) + .ok_or_else(|| PyIndexError::new_err(INPUT_CLASSES_UNSET)) + } + + pub fn inferred_column<'py>( + &self, + py: Python<'py>, + classes: &CompiledViewClasses<'py>, + row: &InferredRow, + ) -> PyResult> { + let (name, column_type, nullability) = row; + if let Some(column) = self + .inferred_column_projections + .lock() + .map_err(|error| native_compiler_error(error.to_string()))? + .get(row) + { + return Ok(column.bind(py).clone()); + } + let fields = PyDict::new(py); + fields.set_item("name", name)?; + fields.set_item("type", column_type.as_deref())?; + fields.set_item("nullability", classes.nullability.call1((nullability,))?)?; + let column = classes.inferred_column.call((), Some(&fields))?; + self.inferred_column_projections + .lock() + .map_err(|error| native_compiler_error(error.to_string()))? + .insert(row.clone(), column.clone().unbind()); + Ok(column) + } +} + +#[pymethods] +impl NativeCompiledProject { + #[new] + pub(crate) fn new() -> Self { + Self::default() + } + + /// Attach a recorded model's typed analysis; `False` when that model was never recorded. + pub fn record_model_analysis( + &mut self, + name: &str, + columns: AnalysisPy, + dynamic_proof: Option, + ) -> bool { + let AnalysisPy { + inferred, + inferred_columns, + nullable_columns, + fast_lineage_has_star, + unchecked, + } = columns; + let analysis: ModelAnalysisFact = ModelAnalysisFact { + inferred_columns, + dynamic_proof: dynamic_proof.map( + |ProofPy { + output_proven, + bare_dynamic_pivot, + families, + failure_reason, + named_families, + }| DynamicProofFact { + output_proven, + bare_dynamic_pivot, + families, + failure_reason, + named_families, + }, + ), + inferred, + nullable_columns, + fast_lineage_has_star, + unchecked_output_columns: unchecked, + }; + record_model_analysis(&mut self.project.facts, name, analysis) + } + + /// Record each assembled native model view's final typed analysis; return how many. + pub fn record_view_analyses( + slf: &Bound<'_, Self>, + views: Vec>, + ) -> PyResult { + let py: Python<'_> = slf.py(); + let mut facts: Vec<(String, ModelAnalysisFact)> = Vec::with_capacity(views.len()); + for view in &views { + facts.push(view.analysis_fact(py)?); + } + let mut owner = slf.borrow_mut(); + let mut recorded: usize = 0; + for (name, analysis) in facts { + recorded += usize::from(record_model_analysis( + &mut owner.project.facts, + &name, + analysis, + )); + } + Ok(recorded) + } + + /// The compile inputs view of this owner's model inputs with these input fields. + pub fn inputs_view( + slf: Bound<'_, Self>, + fields: Bound<'_, PyDict>, + ) -> PyResult { + NativeInputsView::new(slf.unbind(), &fields) + } + + /// The compiled project view of this owner's models with these project fields. + pub fn project_view( + slf: Bound<'_, Self>, + fields: Bound<'_, PyDict>, + ) -> PyResult { + NativeProjectView::new(slf.unbind(), &fields) + } + + /// Pair seed declarations with their CSV files and keep them; return each seed's view. + pub fn attach_seeds( + slf: &Bound<'_, Self>, + declarations: Vec<(Py, Py)>, + seed_files: Vec>, + compile_error: &Bound<'_, PyAny>, + ) -> PyResult> { + let seeds: Vec = paired_seeds(declarations, &seed_files, compile_error)?; + let count: usize = seeds.len(); + slf.borrow_mut().project.seeds = seeds; + Ok((0..count) + .map(|index| NativeSeedInput::new(slf.clone().unbind(), index)) + .collect()) + } + + /// Keep the destinations native project assembly resolved, in seed order. + pub(crate) fn record_seed_namespaces( + &mut self, + namespaces: Vec, + ) -> PyResult<()> { + for (index, namespace) in namespaces.into_iter().enumerate() { + self.project + .seeds + .get_mut(index) + .ok_or_else(|| PyIndexError::new_err(index))? + .set_namespace(namespace); + } + Ok(()) + } + + /// Attach every discovered source in order with the kept loop; return each source's view. + pub fn attach_sources( + slf: &Bound<'_, Self>, + rows: Vec, + settings: SourceSettings, + ) -> PyResult> { + (resource_attachment()?.sources)(slf, rows, settings) + } + + /// Attach every SQL, then Python, function with the kept loop; return each one's view. + pub fn attach_functions( + slf: &Bound<'_, Self>, + rows: (Vec, Vec), + settings: FunctionSettings, + ) -> PyResult> { + (resource_attachment()?.functions)(slf, rows, settings) + } + + /// The orphaned audit factory warnings, missing descriptions and cursor-input errors. + pub(crate) fn resource_diagnostics<'py>( + &self, + py: Python<'py>, + settings: DiagnosticSettings, + ) -> PyResult> { + (resource_attachment()?.diagnostics)(self, py, &settings) + } + + /// Attach every SQL test case, then every scenario, with the kept loop. + pub fn attach_sql_tests( + slf: &Bound<'_, Self>, + files: Vec, + scenarios: Vec, + settings: SqlTestSettings, + ) -> PyResult { + (resource_attachment()?.sql_tests)(slf, files, scenarios, settings) + } + + /// Attach singular, then resource-attached, audits; return views and override warnings. + pub fn attach_audits<'py>( + slf: &Bound<'py, Self>, + files: Vec, + settings: AuditSettings, + ) -> PyResult<(Vec, Vec>)> { + (resource_attachment()?.audits)(slf, files, settings) + } + /// Keep the scope dependency keys native project assembly resolved, in audit order. + pub(crate) fn record_audit_scope_deps(&mut self, deps: Vec>) -> PyResult<()> { + for (index, keys) in deps.into_iter().enumerate() { + self.project + .audits + .get_mut(index) + .ok_or_else(|| PyIndexError::new_err(index))? + .set_scope_deps(keys); + } + Ok(()) + } + + /// The assembled audit at `index`. + pub(crate) fn compiled_audit( + slf: &Bound<'_, Self>, + index: usize, + ) -> PyResult { + if index >= slf.borrow().project.audits.len() { + return Err(PyIndexError::new_err(index)); + } + Ok(NativeCompiledAudit::new(slf.clone().unbind(), index)) + } + + /// Keep the dependency keys native project assembly resolved, in function order. + pub(crate) fn record_function_deps(&mut self, deps: Vec>) -> PyResult<()> { + for (index, keys) in deps.into_iter().enumerate() { + self.project + .functions + .get_mut(index) + .ok_or_else(|| PyIndexError::new_err(index))? + .set_deps(keys); + } + Ok(()) + } + + /// The assembled function at `index`. + pub(crate) fn compiled_function( + slf: &Bound<'_, Self>, + index: usize, + ) -> PyResult { + if index >= slf.borrow().project.functions.len() { + return Err(PyIndexError::new_err(index)); + } + Ok(NativeCompiledFunction::new(slf.clone().unbind(), index)) + } + + /// Keep the namespaces native project assembly resolved, in source order. + pub(crate) fn record_source_namespaces( + &mut self, + namespaces: Vec, + ) -> PyResult<()> { + for (index, namespace) in namespaces.into_iter().enumerate() { + self.project + .sources + .get_mut(index) + .ok_or_else(|| PyIndexError::new_err(index))? + .set_namespace(namespace); + } + Ok(()) + } + + /// The assembled source at `index`. + pub(crate) fn compiled_source( + slf: &Bound<'_, Self>, + index: usize, + ) -> PyResult { + if index >= slf.borrow().project.sources.len() { + return Err(PyIndexError::new_err(index)); + } + Ok(NativeCompiledSource::new(slf.clone().unbind(), index)) + } + + /// The assembled seed at `index`. + pub(crate) fn compiled_seed( + slf: &Bound<'_, Self>, + index: usize, + ) -> PyResult { + if index >= slf.borrow().project.seeds.len() { + return Err(PyIndexError::new_err(index)); + } + Ok(NativeCompiledSeed::new(slf.clone().unbind(), index)) + } + + /// How many lineage graphs this owner's project views built. + pub(crate) fn graph_builds(&self) -> usize { + self.project.graph_builds + } + + /// Whether both projects retained the same facts. + pub(crate) fn __eq__(&self, other: &Self) -> bool { + self.project.facts == other.project.facts + } + + /// The recorded models' names, in production order. + pub(crate) fn model_names(&self) -> Vec { + project_models(&self.project.facts) + .iter() + .map(|model| model.name.clone()) + .collect() + } +} + +/// A reference as the tuple Python reads: kind, name, package and call argument count. +#[derive(Debug)] +pub struct ReferenceTuple { + pub kind: String, + pub name: String, + pub package: Option, + pub call_argument_count: Option, +} + +impl<'py> pyo3::IntoPyObject<'py> for ReferenceTuple { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject( + (self.kind, self.name, self.package, self.call_argument_count), + py, + ) + } +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/types.rs b/crates/sqlbuild-python-project/src/compiled_project/types.rs new file mode 100644 index 0000000000..9254d9b6ea --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/types.rs @@ -0,0 +1,48 @@ +//! Public traits and type aliases of compiled project. + +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; + +/// One discovered source: its file's path, the discovered file and the entry. +pub type SourceRow = (String, Py, Py); + +/// `(default contract, whether expressions are syntax-checked, source kind)`. +pub type SourceSettings = (Option, bool, Py); + +/// A test case's discovered case, name and parameter values in authored order. +pub type TestCaseRow = (Py, String, Vec<(String, Py)>); + +/// The attached SQL test inputs, then scenario inputs. +pub type AttachedSqlTests = (Vec>, Vec>); + +/// A loader's description, relative path and whether that path is a YAML file. +pub type LoaderRow = (Option, String, bool); + +/// The orphaned audit factory warnings, then the missing descriptions, then the cursor errors. +pub type ResourceDiagnostics<'py> = ( + Vec>, + Vec>, + Vec>, +); + +/// The tuple Python sends for a `FunctionSettings`, in field order. +pub type FunctionSettingsTuple = ( + String, + bool, + (Option, Option), + Option<(Option, Option)>, + bool, + Py, +); + +/// The tuple Python sends for a `TestBlockRow`, in field order. +pub type TestBlockTuple = ( + Py, + Option, + String, + String, + Vec, + Vec, + Option<(usize, usize)>, +); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_analysis.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_analysis.rs index ca2d959b72..dd703c2c3f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_analysis.rs @@ -1,12 +1,12 @@ //! A compiled model's analysis, kept from the native session and projected on demand. +use crate::compiled_views::models::AnalysisProjections; use sqlbuild_python_base::positional_record; use std::borrow::Cow; use std::collections::HashMap; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{IntoPyObject, IntoPyObjectExt}; use sqlbuild_analysis::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql; @@ -26,11 +26,12 @@ use sqlbuild_analysis::sql_macros::models::SqlMacroFinding; use sqlbuild_project::models as project_models; use sqlbuild_project::types::InferredRow; -use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; -use crate::bindings::_helpers::compiled_views::compiled_model_view::CompiledViewClasses; -use crate::bindings::_helpers::compiled_views::model_input_view::ModelInputRecord; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::lineage::lineage_facts::{confidence_member, transform_kind_member}; +use crate::analysis_session::models::NativeModelAnalysisSession; +use crate::compiled_views::models::CompiledViewClasses; +use crate::compiled_views::models::PyProjectHost; +use crate::compiled_views::types::ModelInputRecord; +use crate::lineage::main::confidence_member::confidence_member; +use crate::lineage::main::transform_kind_member::transform_kind_member; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::types::RecordId; @@ -61,23 +62,6 @@ pub(crate) type SessionArgument = (Py, usize); /// What one model's analysis left for its compiled model. pub(crate) type AnalysisRecord = project_models::AnalysisRecord; -/// Python projections of the analysis, each built once on first access. -pub(crate) struct AnalysisProjections { - lineage: PyOnceLock>, - binding: PyOnceLock>, - proof: PyOnceLock>, -} - -impl Default for AnalysisProjections { - fn default() -> Self { - Self { - lineage: PyOnceLock::new(), - binding: PyOnceLock::new(), - proof: PyOnceLock::new(), - } - } -} - fn inferred_row(column: &ColumnFact) -> InferredRow { ( column.name.clone(), diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs new file mode 100644 index 0000000000..dca81a3464 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs @@ -0,0 +1,167 @@ +//! A compiled model, read from the native compiled project that owns it. + +use crate::compiled_views::models::CompiledProjections; +use crate::compiled_views::types::LineageInput; +use sqlbuild_project::models as project_models; +use sqlbuild_project::models::DestinationFact; +use sqlbuild_project::types::InferredRow; +use sqlbuild_python_front::model_config::models::HeaderAudits; +use std::sync::Arc; + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +use sqlbuild_analysis::assembly::analysis_session::models::LineageRow; + +use crate::compiled_views::_helpers::compiled_models::compiled_model_analysis::{ + AnalysisRecord, ProofArgument, SessionArgument, analysis_record, +}; +use crate::compiled_views::models::PyProjectHost; +use crate::compiled_views::types::ModelInputRecord; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::model_config::models::HeaderColumns; + +pub(crate) const DATABASE_KEY: &str = "database"; +pub(crate) const SCHEMA_KEY: &str = "schema"; +const ALIAS_KEY: &str = "alias"; +pub(crate) const VALUES_ATTRIBUTE: &str = "values"; +pub(crate) const LOGICAL_SCHEMA_ATTRIBUTE: &str = "logical_schema"; +pub(crate) const LOGICAL_DATABASE_ATTRIBUTE: &str = "logical_database"; +pub(crate) const MODEL_RESOURCE: &str = "model"; +pub(crate) const SQL_ANALYSIS_KEY: &str = "sql_analysis"; +pub(crate) const TAGS_KEY: &str = "tags"; +pub(crate) const NULLABLE: &str = "nullable"; + +/// A schema's entry audits, column specs and each audited column's index and name. +pub(crate) type AuditedSchema = (Arc, Arc, Vec<(usize, String)>); + +/// `(database, schema, name)` of a relation. +pub(crate) type DestinationRow = (Option, Option, String); + +/// What assembly and the semantic checks finished for one compiled model. +pub(crate) type CompiledRecord = project_models::CompiledRecord; + +/// What later stages changed for one view of a model; the owner's record stays as assembled. +#[derive(Debug, Default)] +pub(crate) struct ModelOverlay { + pub(crate) binding_diagnostics: Option>, + /// Output columns whose types semantic recovery forgot, left unchecked by the contract. + pub(crate) poisoned: Vec, + pub(crate) destination: Option, +} + +impl ModelOverlay { + pub(crate) fn clone_ref(&self, py: Python<'_>) -> Self { + Self { + binding_diagnostics: self + .binding_diagnostics + .as_ref() + .map(|diagnostics| diagnostics.clone_ref(py)), + poisoned: self.poisoned.clone(), + destination: self.destination.clone(), + } + } +} + +/// One view's projections of attributes its overlay can change, each built once. +pub(crate) struct ViewProjections { + pub(crate) destination: PyOnceLock>, + pub(crate) inferred_columns: PyOnceLock>, +} + +impl std::fmt::Debug for ViewProjections { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ViewProjections") + .finish_non_exhaustive() + } +} + +impl Default for ViewProjections { + fn default() -> Self { + Self { + destination: PyOnceLock::new(), + inferred_columns: PyOnceLock::new(), + } + } +} + +/// Deps, session analysis, failed analysis's columns, standalone proof, and macro deps. +#[derive(FromPyObject)] +pub(crate) struct CompiledArguments( + Vec<(String, String)>, + Option, + Option>, + Option, + Vec, +); + +/// The compiled record of one model, with its configured destination. +pub(crate) fn compiled_record( + py: Python<'_>, + input: &ModelInputRecord, + name: &str, + arguments: CompiledArguments, +) -> PyResult { + let CompiledArguments(deps, session, fallback, standalone_proof, macro_deps) = arguments; + let analysis: AnalysisRecord = + analysis_record(py, input, session, (fallback, standalone_proof))?; + let config: &Bound<'_, PyAny> = input.host.config.bind(py); + let values: Bound<'_, PyDict> = native_field(config, VALUES_ATTRIBUTE)?.downcast_into()?; + Ok(CompiledRecord { + deps, + destination: DestinationFact { + database: text_value(&values, DATABASE_KEY)?, + schema: text_value(&values, SCHEMA_KEY)?, + name: text_value(&values, ALIAS_KEY)?.unwrap_or_else(|| name.to_owned()), + qualified_name: None, + logical_schema: native_field(config, LOGICAL_SCHEMA_ATTRIBUTE)?.extract()?, + logical_database: native_field(config, LOGICAL_DATABASE_ATTRIBUTE)?.extract()?, + }, + analysis, + macro_deps, + host: CompiledProjections::default(), + }) +} + +fn text_value(values: &Bound<'_, PyDict>, key: &str) -> PyResult> { + match values.get_item(key)? { + Some(value) if value.is_instance_of::() => { + Ok(Some(value.extract()?)) + } + _ => Ok(None), + } +} + +/// A lineage row's output column with its `(resource name, column)` sources. +pub(crate) fn lineage_input(row: &LineageRow) -> LineageInput { + let mut sources: Vec<(String, String)> = Vec::with_capacity(row.sources.len()); + for (_, resource, column) in &row.sources { + sources.push((resource.clone(), column.clone())); + } + (row.output_column.clone(), sources) +} + +/// Each family's name casefolded as Python's, with its inferred type. +pub(crate) fn casefolded_families( + families: &[(String, Option)], +) -> Vec<(String, Option)> { + families + .iter() + .map(|(name, kind)| (str::to_lowercase(name), kind.clone())) + .collect() +} + +/// The compiled record of an input, which assembly records before any view is handed out. +pub(crate) fn compiled_of(record: &ModelInputRecord) -> PyResult<&CompiledRecord> { + record + .compiled + .as_ref() + .ok_or_else(|| pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled")) +} + +pub(crate) fn model_name(record: &ModelInputRecord) -> String { + record.name.clone() +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/mod.rs new file mode 100644 index 0000000000..8f5cc23169 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod compiled_model_analysis; +pub(crate) mod compiled_model_view; +pub(crate) mod model_reads; +pub(crate) mod relation_keys; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/model_reads.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/model_reads.rs new file mode 100644 index 0000000000..9524176ce3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/model_reads.rs @@ -0,0 +1,26 @@ +//! A compiled model's analysis outcomes as Rust data: views read their owner, records their fields. + +use pyo3::prelude::PyResult; + +use sqlbuild_project::types::InferredRow; + +pub(crate) const TRANSFORM_VALUES: [&str; 6] = [ + "direct", + "cast", + "expression", + "aggregation", + "star", + "constant", +]; +pub(crate) const CONFIDENCE_VALUES: [&str; 3] = ["unknown", "high", "medium"]; + +pub(crate) fn column_names(rows: Vec) -> Vec { + rows.into_iter().map(|(name, _, _)| name).collect() +} + +pub(crate) fn coded(values: &[&str], code: u8) -> PyResult { + values + .get(usize::from(code)) + .map(|value| (*value).to_owned()) + .ok_or_else(|| pyo3::exceptions::PyIndexError::new_err("tuple index out of range")) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/relation_keys.rs new file mode 100644 index 0000000000..b12a75d8cd --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/relation_keys.rs @@ -0,0 +1,197 @@ +//! Declared unique keys of project relations, derived natively from their key facts. + +use crate::compiled_views::main::compiled_models::project_relation_keys::project_relation_keys; +use crate::compiled_views::main::compiled_models::relation_keys::relation_keys; +use crate::compiled_views::models::RelationKeyRow; +use crate::compiled_views::types::RelationKeys; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; +use sqlbuild_rules::relation_keys::models::{DeclaredKeyColumn, DeclaredRelation}; + +use crate::resource_views::main::source_views::read_source_view::read_source_view; +use crate::resource_views::models::SourceKeyFacts; +use crate::resource_views::types::SourceViews; + +const MATERIALIZED_CONFIG: &str = "materialized"; +const INCREMENTAL_MODE_CONFIG: &str = "incremental_mode"; +const INCREMENTAL_STRATEGY_CONFIG: &str = "incremental_strategy"; +const CURSOR_CONFIG: &str = "cursor"; +const UNIQUE_KEY_CONFIG: &str = "unique_key"; +pub(crate) const UNIQUE_AUDIT_COLUMN_ARGUMENT: &str = "column"; +pub(crate) const RELATION_KIND_SEED: &str = "seed"; +pub(crate) const RELATION_KIND_SOURCE: &str = "source"; + +/// Native declared columns and model audits used to derive relation keys. +type DeclaredKeyFacts<'a> = (&'a [SchemaColumnFact], Vec<(String, Option)>); + +/// The key row of one relation from its config values and declared column and audit specs. +pub(crate) fn relation_key_row( + kind: &str, + name: String, + values: &Bound<'_, PyDict>, + declared: Option>, +) -> PyResult { + let mut columns: Vec<(String, Vec, bool)> = Vec::new(); + let mut audits: Vec<(String, Option)> = Vec::new(); + if let Some((declared_columns, declared_audits)) = declared { + for column in declared_columns { + columns.push(( + column.name.clone(), + column.audit_definitions.clone(), + column.nullable == Some(false), + )); + } + audits = declared_audits; + } + Ok(RelationKeyRow { + kind: kind.to_owned(), + name, + materialized: config_text(values, MATERIALIZED_CONFIG)?, + mode: config_text(values, INCREMENTAL_MODE_CONFIG)?, + strategy: config_text(values, INCREMENTAL_STRATEGY_CONFIG)?, + has_cursor: values + .get_item(CURSOR_CONFIG)? + .is_some_and(|value| value.is_truthy().unwrap_or(false)), + unique_key: unique_key(values)?, + columns, + audits, + }) +} + +/// A source's key row from its native view's declared facts, or `None` for any other object. +pub(crate) fn source_view_key_row(view: &Bound<'_, PyAny>) -> Option> { + read_source_view(view, |source, py| { + let facts: &SourceKeyFacts = source.key_facts(py)?; + Ok(RelationKeyRow { + kind: RELATION_KIND_SOURCE.to_owned(), + name: source.attached().name.clone(), + materialized: String::new(), + mode: String::new(), + strategy: String::new(), + has_cursor: false, + unique_key: None, + columns: facts + .columns + .iter() + .map(|column| { + ( + column.name.clone(), + column.audit_definitions.clone(), + column.nullable == Some(false), + ) + }) + .collect(), + audits: facts.audits.clone(), + }) + }) +} + +/// `str(values.get(key, ""))`. +fn config_text(values: &Bound<'_, PyDict>, key: &str) -> PyResult { + match values.get_item(key)? { + Some(value) => value.str()?.extract(), + None => Ok(String::new()), + } +} + +/// `unique_key` as a name list, when it is a name or a non-empty list or tuple of names. +pub(crate) fn unique_key(values: &Bound<'_, PyDict>) -> PyResult>> { + let Some(value) = values.get_item(UNIQUE_KEY_CONFIG)? else { + return Ok(None); + }; + if value.is_instance_of::() { + return Ok(Some(vec![value.extract()?])); + } + if !(value.is_instance_of::() || value.is_instance_of::()) { + return Ok(None); + } + let mut names: Vec = Vec::new(); + for item in value.try_iter()? { + let item: Bound<'_, PyAny> = item?; + if !item.is_instance_of::() { + return Ok(None); + } + names.push(item.str()?.extract()?); + } + Ok(Some(names)) +} + +/// `getattr(entry, name, None) or ()`. +pub(crate) fn declared_specs<'py>( + entry: &Bound<'py, PyAny>, + name: &str, +) -> PyResult> { + match entry.getattr_opt(name)? { + Some(specs) if specs.is_truthy()? => Ok(specs), + _ => Ok(PyTuple::empty(entry.py()).into_any()), + } +} + +/// `keys` as a `{identity: ((column, ...), ...)}` dict. +fn keys_dict<'py>(py: Python<'py>, keys: RelationKeys) -> PyResult> { + let found: Bound<'py, PyDict> = PyDict::new(py); + for (identity, unique_keys) in keys { + let mut tuples: Vec> = Vec::with_capacity(unique_keys.len()); + for key in unique_keys { + tuples.push(PyTuple::new(py, key)?); + } + found.set_item(identity, PyTuple::new(py, tuples)?)?; + } + Ok(found) +} + +/// Keys that prove one row per key, by `kind:name` identity, from model rows, seeds and sources. +#[pyfunction] +#[pyo3(signature = (*, model_rows, seeds, sources))] +pub(crate) fn declared_relation_keys<'py>( + py: Python<'py>, + model_rows: Vec, + seeds: &Bound<'py, PyAny>, + sources: &Bound<'py, PyAny>, +) -> PyResult> { + keys_dict(py, relation_keys(py, model_rows, (seeds, sources), &[])?) +} + +/// Declared keys of a compiled project's models, seeds and sources. +#[pyfunction] +pub(crate) fn compiled_relation_keys<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult> { + keys_dict(project.py(), project_relation_keys(project)?) +} + +pub(crate) fn declared_relation(row: RelationKeyRow) -> DeclaredRelation { + let RelationKeyRow { + kind, + name, + materialized, + mode, + strategy, + has_cursor, + unique_key, + columns, + audits, + } = row; + DeclaredRelation { + kind, + name, + materialized, + incremental_mode: mode, + incremental_strategy: strategy, + has_cursor, + unique_key, + columns: columns + .into_iter() + .map(|(name, audits, not_nullable)| DeclaredKeyColumn { + name, + audits, + not_nullable, + }) + .collect(), + audits, + at_most_one_row: false, + } +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/mod.rs new file mode 100644 index 0000000000..e0b40801cf --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod compiled_models; +pub(crate) mod model_inputs; +pub(crate) mod project; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/inputs_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/inputs_view.rs new file mode 100644 index 0000000000..ffb2686b6d --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/inputs_view.rs @@ -0,0 +1,189 @@ +//! A compile's attached inputs, read from the native compiled project that owns its models. + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +const MODEL_INPUTS_FIELD: &str = "model_inputs"; +const RUN_ID_FIELD: &str = "run_id"; +const NO_SQL_VALIDATION_FIELD: &str = "no_sql_validation"; + +/// Compile inputs held as Python objects: user code, discovery or an integration owns them. +pub(crate) struct BoundaryInputs { + /// The discovered project config. + pub(crate) project_config: Py, + /// The discovered local config. + pub(crate) local_config: Py, + /// The discovered project inputs. + pub(crate) discovered_inputs: Py, + /// The adapter's lexical syntax. + pub(crate) sql_lexical_syntax: Py, + /// The selected target config, or `None`. + pub(crate) effective_target: Py, + /// The compile cache directory path. + pub(crate) compile_cache_dir: Py, + /// The resolved connection, adapter integration values. + pub(crate) effective_connection: Py, + /// Project variables, user values. + pub(crate) effective_vars: Py, + /// The macro context user macro bodies run with. + pub(crate) macro_context: Py, + /// Loaded macro modules, user code. + pub(crate) loaded_macros: Py, + /// Public enums, user declaration values. + pub(crate) public_enums: Py, + /// Public constants, user declaration values. + pub(crate) public_constants: Py, + /// The external reference resolver integration, or `None`. + pub(crate) external_sql_reference_resolver: Py, +} + +/// Compiler-derived inputs still held as Python objects until they are owner facts. +pub(crate) struct CompilerHandles { + /// Effective settings. + pub(crate) effective_settings: Py, + /// Attached seed inputs. + pub(crate) seed_inputs: Py, + /// Attached source inputs. + pub(crate) source_inputs: Py, + /// Attached SQL function inputs. + pub(crate) sql_function_inputs: Py, + /// Attached SQL test inputs. + pub(crate) test_inputs: Py, + /// Attached SQL scenario inputs. + pub(crate) scenario_inputs: Py, + /// Attached audit inputs. + pub(crate) audit_inputs: Py, + /// Compiler diagnostics. + pub(crate) diagnostics: Py, + /// The declaration scope index. + pub(crate) scope_index: Py, + /// The declaration scope build. + pub(crate) declaration_scope: Py, +} + +/// The values of one compile inputs view. +pub(crate) struct InputsFields { + pub(crate) run_id: String, + pub(crate) effective_target_name: Option, + pub(crate) no_sql_validation: bool, + pub(crate) model_inputs: Py, + pub(crate) boundary: BoundaryInputs, + pub(crate) compiler: CompilerHandles, +} + +impl std::fmt::Debug for InputsFields { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("InputsFields") + .finish_non_exhaustive() + } +} + +impl InputsFields { + /// The fields named in `fields`, each required. + pub(crate) fn from_fields(fields: &Bound<'_, PyDict>) -> PyResult { + Ok(Self { + run_id: required(fields, RUN_ID_FIELD)?.extract()?, + effective_target_name: required(fields, "effective_target_name")?.extract()?, + no_sql_validation: required(fields, NO_SQL_VALIDATION_FIELD)?.extract()?, + model_inputs: required(fields, MODEL_INPUTS_FIELD)? + .downcast_into::()? + .unbind(), + boundary: BoundaryInputs { + project_config: required(fields, "project_config")?.unbind(), + local_config: required(fields, "local_config")?.unbind(), + discovered_inputs: required(fields, "discovered_inputs")?.unbind(), + sql_lexical_syntax: required(fields, "sql_lexical_syntax")?.unbind(), + effective_target: required(fields, "effective_target")?.unbind(), + compile_cache_dir: required(fields, "compile_cache_dir")?.unbind(), + effective_connection: required(fields, "effective_connection")?.unbind(), + effective_vars: required(fields, "effective_vars")?.unbind(), + macro_context: required(fields, "macro_context")?.unbind(), + loaded_macros: required(fields, "loaded_macros")?.unbind(), + public_enums: required(fields, "public_enums")?.unbind(), + public_constants: required(fields, "public_constants")?.unbind(), + external_sql_reference_resolver: required( + fields, + "external_sql_reference_resolver", + )? + .unbind(), + }, + compiler: CompilerHandles { + effective_settings: required(fields, "effective_settings")?.unbind(), + seed_inputs: required(fields, "seed_inputs")?.unbind(), + source_inputs: required(fields, "source_inputs")?.unbind(), + sql_function_inputs: required(fields, "sql_function_inputs")?.unbind(), + test_inputs: required(fields, "test_inputs")?.unbind(), + scenario_inputs: required(fields, "scenario_inputs")?.unbind(), + audit_inputs: required(fields, "audit_inputs")?.unbind(), + diagnostics: required(fields, "diagnostics")?.unbind(), + scope_index: required(fields, "scope_index")?.unbind(), + declaration_scope: required(fields, "declaration_scope")?.unbind(), + }, + }) + } + + /// Every field by name. + pub(crate) fn as_fields<'py>(&self, py: Python<'py>) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item(RUN_ID_FIELD, &self.run_id)?; + fields.set_item("effective_target_name", &self.effective_target_name)?; + fields.set_item(NO_SQL_VALIDATION_FIELD, self.no_sql_validation)?; + fields.set_item(MODEL_INPUTS_FIELD, self.model_inputs.bind(py))?; + fields.set_item("project_config", self.boundary.project_config.bind(py))?; + fields.set_item("local_config", self.boundary.local_config.bind(py))?; + fields.set_item( + "discovered_inputs", + self.boundary.discovered_inputs.bind(py), + )?; + fields.set_item( + "sql_lexical_syntax", + self.boundary.sql_lexical_syntax.bind(py), + )?; + fields.set_item("effective_target", self.boundary.effective_target.bind(py))?; + fields.set_item( + "compile_cache_dir", + self.boundary.compile_cache_dir.bind(py), + )?; + fields.set_item( + "effective_connection", + self.boundary.effective_connection.bind(py), + )?; + fields.set_item("effective_vars", self.boundary.effective_vars.bind(py))?; + fields.set_item("macro_context", self.boundary.macro_context.bind(py))?; + fields.set_item("loaded_macros", self.boundary.loaded_macros.bind(py))?; + fields.set_item("public_enums", self.boundary.public_enums.bind(py))?; + fields.set_item("public_constants", self.boundary.public_constants.bind(py))?; + fields.set_item( + "external_sql_reference_resolver", + self.boundary.external_sql_reference_resolver.bind(py), + )?; + fields.set_item( + "effective_settings", + self.compiler.effective_settings.bind(py), + )?; + fields.set_item("seed_inputs", self.compiler.seed_inputs.bind(py))?; + fields.set_item("source_inputs", self.compiler.source_inputs.bind(py))?; + fields.set_item( + "sql_function_inputs", + self.compiler.sql_function_inputs.bind(py), + )?; + fields.set_item("test_inputs", self.compiler.test_inputs.bind(py))?; + fields.set_item("scenario_inputs", self.compiler.scenario_inputs.bind(py))?; + fields.set_item("audit_inputs", self.compiler.audit_inputs.bind(py))?; + fields.set_item("diagnostics", self.compiler.diagnostics.bind(py))?; + fields.set_item("scope_index", self.compiler.scope_index.bind(py))?; + fields.set_item( + "declaration_scope", + self.compiler.declaration_scope.bind(py), + )?; + Ok(fields) + } +} + +fn required<'py>(fields: &Bound<'py, PyDict>, name: &str) -> PyResult> { + fields + .get_item(name)? + .ok_or_else(|| PyTypeError::new_err(format!("missing compile input field {name}"))) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/mod.rs new file mode 100644 index 0000000000..c4dc7d398f --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod inputs_view; +pub(crate) mod model_input_view; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs new file mode 100644 index 0000000000..0046c3ab7b --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs @@ -0,0 +1,191 @@ +//! A model's compile input, read from the native compiled project that owns it. + +use crate::compiled_views::main::model_inputs::reference_tuple::reference_tuple; +use crate::compiled_views::models::InputViewClasses; +use crate::compiled_views::types::ModelInputRecord; +use sqlbuild_project::models::ExpansionRecord; +use std::sync::Arc; + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; + +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::_helpers::compiled_models::relation_keys::relation_key_row; +use crate::compiled_views::models::RelationKeyRow; +use sqlbuild_project::models::ReferenceRow; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::main::records::projected_text::projected_text; +use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::render::main::calls::span_tuple::span_tuple; +use sqlbuild_python_front::render::types::SpanRow; + +const MODEL_RELATION_KIND: &str = "ref"; + +/// Read the input record at `index` of `owner` together with `owner`. +pub(crate) fn read_input_with_owner( + owner: &Py, + index: usize, + py: Python<'_>, + read: impl FnOnce(&NativeCompiledProject, &ModelInputRecord) -> PyResult, +) -> PyResult { + let owner = owner.bind(py).borrow(); + read(&owner, owner.input(index)?) +} + +/// Read the input record at `index` of `owner`. +pub(crate) fn read_input<'py, T>( + owner: &Py, + index: usize, + py: Python<'py>, + read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, +) -> PyResult { + let owner = owner.bind(py).borrow(); + read(owner.input(index)?, py) +} + +/// The compiled query as the Python string every reader shares; the authored query's when equal. +pub(crate) fn query_sql_text<'py>( + py: Python<'py>, + record: &ModelInputRecord, +) -> Bound<'py, PyString> { + let model_file: &NativeModelFile = record.host.model_file.get(); + if Arc::ptr_eq(&record.query_sql, model_file.query_text()) { + return model_file.query_sql(py); + } + projected_text(py, &record.host.projections.query_sql, &record.query_sql) +} + +/// The authored-to-expanded passes, when interpolation left the authored query unchanged. +pub(crate) fn authored_expansion( + classes: &InputViewClasses<'_>, + record: &ModelInputRecord, +) -> PyResult> { + let py: Python<'_> = classes.sql_expansion.py(); + record + .host + .projections + .sql_expansion + .get_or_try_init(py, || { + let expansion: &ExpansionRecord = &record.expansion; + if !expansion.authored { + return Ok(py.None()); + } + expansion_object( + classes, + expansion, + &[&expansion.declaration_spans, &expansion.macro_spans], + ) + }) + .map(|expansion| expansion.clone_ref(py)) +} + +/// Where a rejected `sql_analysis false` was authored, or `None`. +pub(crate) fn rejected_opt_out(record: &ModelInputRecord, py: Python<'_>) -> Py { + record + .host + .rejected_opt_out + .as_ref() + .map_or_else(|| py.None(), |location| location.clone_ref(py)) +} + +/// The model's `CompileSqlReference` tuple, built once. +pub(crate) fn references( + classes: &InputViewClasses<'_>, + record: &ModelInputRecord, +) -> PyResult> { + reference_tuple( + classes, + &record.references, + &record.host.projections.references, + ) +} + +/// The model's declared key facts: its config and attached schema columns and audits. +pub(crate) fn model_relation_key_row( + owner: &NativeCompiledProject, + record: &ModelInputRecord, + py: Python<'_>, +) -> PyResult { + let values: Bound<'_, PyAny> = native_field(record.host.config.bind(py), "values")?; + let declared = match (&record.schema, owner.input_schema_fact(record)) { + (Some(schema), Some(fact)) => { + Some((fact.columns.as_slice(), schema.host.audits.key_rows(py)?)) + } + _ => None, + }; + relation_key_row( + MODEL_RELATION_KIND, + record.name.clone(), + values.downcast::()?, + declared, + ) +} + +/// The name of the model's attached schema entry, its file stem, or `None`. +pub(crate) fn attached_schema_name(record: &ModelInputRecord) -> Option { + record.schema.as_ref().map(|_| record.name.clone()) +} + +/// The model's attached `SchemaModelEntry`, or `None`, built once. +pub(crate) fn schema_entry( + classes: &InputViewClasses<'_>, + record: &ModelInputRecord, +) -> PyResult> { + let py: Python<'_> = classes.schema_model_entry.py(); + record + .host + .projections + .schema_entry + .get_or_try_init(py, || { + let Some(schema) = &record.schema else { + return Ok::<_, pyo3::PyErr>(py.None()); + }; + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("name", &record.name)?; + fields.set_item("model_schema", &schema.model_schema)?; + fields.set_item("description", &schema.description)?; + fields.set_item("type_enforcement", schema.type_enforcement.then_some(true))?; + fields.set_item("columns", schema.host.columns.project(py)?)?; + fields.set_item("dynamic_columns", schema.host.dynamic_columns.project(py)?)?; + fields.set_item("audits", schema.host.audits.project(py)?)?; + Ok(classes.schema_model_entry.call((), Some(&fields))?.unbind()) + }) + .map(|entry| entry.clone_ref(py)) +} + +/// `CompiledSqlExpansion(authored_sql, expanded_sql, passes)` for these passes. +pub(crate) fn expansion_object( + classes: &InputViewClasses<'_>, + expansion: &ExpansionRecord, + passes: &[&Vec], +) -> PyResult> { + let py: Python<'_> = classes.sql_expansion.py(); + let mut pass_objects: Vec> = Vec::with_capacity(passes.len()); + for spans in passes { + let mut span_objects: Vec> = Vec::with_capacity(spans.len()); + for span in spans.iter() { + span_objects.push( + classes + .expansion_span + .call1(span_tuple(span).into_pyobject(py)?)?, + ); + } + pass_objects.push(PyTuple::new(py, span_objects)?); + } + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("authored_sql", &*expansion.authored_sql)?; + fields.set_item("expanded_sql", &*expansion.expanded_sql)?; + fields.set_item("passes", PyTuple::new(py, pass_objects)?)?; + Ok(classes.sql_expansion.call((), Some(&fields))?.unbind()) +} + +/// One `CompileSqlReference` record as its row. +pub(crate) fn reference_row_of(reference: &Bound<'_, PyAny>) -> PyResult { + Ok(ReferenceRow { + kind: native_field(reference, "ref_kind")?.str()?.extract()?, + name: native_field(reference, "ref_name")?.extract()?, + package: native_field(reference, "ref_package")?.extract()?, + call_argument_count: native_field(reference, "call_argument_count")?.extract()?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/getters.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/getters.rs rename to crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs index 6da8e57153..51ed97700f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/getters.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs @@ -3,76 +3,16 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::compiled_views::audit_views::{ - NativeAuditInput, NativeCompiledAudit, -}; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::function_views::{ - NativeCompiledFunction, NativeFunctionInput, -}; -use crate::bindings::_helpers::compiled_views::inputs_view::NativeInputsView; -use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::compiled_views::seed_views::{NativeCompiledSeed, NativeSeedInput}; -use crate::bindings::_helpers::compiled_views::source_views::{ - NativeCompiledSource, NativeSourceInput, -}; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeInputsView; +use crate::compiled_views::models::NativeModelInput; +use crate::compiled_views::models::NativeProjectView; +use crate::resource_views::models::{NativeAuditInput, NativeCompiledAudit}; +use crate::resource_views::models::{NativeCompiledFunction, NativeFunctionInput}; +use crate::resource_views::models::{NativeCompiledSeed, NativeSeedInput}; +use crate::resource_views::models::{NativeCompiledSource, NativeSourceInput}; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[ - GetterEntry { - class: class_of::, - read: read_native_audit_input, - }, - GetterEntry { - class: class_of::, - read: read_native_compiled_audit, - }, - GetterEntry { - class: class_of::, - read: read_native_compiled_model, - }, - GetterEntry { - class: class_of::, - read: read_native_compiled_function, - }, - GetterEntry { - class: class_of::, - read: read_native_function_input, - }, - GetterEntry { - class: class_of::, - read: read_native_inputs_view, - }, - GetterEntry { - class: class_of::, - read: read_native_model_input, - }, - GetterEntry { - class: class_of::, - read: read_native_project_view, - }, - GetterEntry { - class: class_of::, - read: read_native_compiled_seed, - }, - GetterEntry { - class: class_of::, - read: read_native_seed_input, - }, - GetterEntry { - class: class_of::, - read: read_native_compiled_source, - }, - GetterEntry { - class: class_of::, - read: read_native_source_input, - }, -]; - -fn read_native_audit_input<'py>( +pub(crate) fn read_native_audit_input<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -150,7 +90,7 @@ fn read_native_audit_input<'py>( }) } -fn read_native_compiled_audit<'py>( +pub(crate) fn read_native_compiled_audit<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -229,7 +169,7 @@ fn read_native_compiled_audit<'py>( }) } -fn read_native_compiled_model<'py>( +pub(crate) fn read_native_compiled_model<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -344,7 +284,7 @@ fn read_native_compiled_model<'py>( }) } -fn read_native_compiled_function<'py>( +pub(crate) fn read_native_compiled_function<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -406,7 +346,7 @@ fn read_native_compiled_function<'py>( }) } -fn read_native_function_input<'py>( +pub(crate) fn read_native_function_input<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -487,7 +427,7 @@ fn read_native_function_input<'py>( }) } -fn read_native_inputs_view<'py>( +pub(crate) fn read_native_inputs_view<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -533,7 +473,7 @@ fn read_native_inputs_view<'py>( }) } -fn read_native_model_input<'py>( +pub(crate) fn read_native_model_input<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -607,7 +547,7 @@ fn read_native_model_input<'py>( }) } -fn read_native_project_view<'py>( +pub(crate) fn read_native_project_view<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -669,7 +609,7 @@ fn read_native_project_view<'py>( }) } -fn read_native_compiled_seed<'py>( +pub(crate) fn read_native_compiled_seed<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -701,7 +641,7 @@ fn read_native_compiled_seed<'py>( }) } -fn read_native_seed_input<'py>( +pub(crate) fn read_native_seed_input<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -726,7 +666,7 @@ fn read_native_seed_input<'py>( }) } -fn read_native_compiled_source<'py>( +pub(crate) fn read_native_compiled_source<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -751,7 +691,7 @@ fn read_native_compiled_source<'py>( }) } -fn read_native_source_input<'py>( +pub(crate) fn read_native_source_input<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/mod.rs new file mode 100644 index 0000000000..bd4765fc84 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod getters; +pub(crate) mod project_host; +pub(crate) mod project_reads; +pub(crate) mod project_view; +pub(crate) mod records; diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_host.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_host.rs new file mode 100644 index 0000000000..269a8bc275 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_host.rs @@ -0,0 +1,75 @@ +//! The Python host's slots beside the compiled project's records. + +use std::fmt::{Debug, Formatter, Result as FmtResult}; + +use crate::compiled_views::models::AnalysisProjections; +use crate::compiled_views::models::CompiledProjections; +use crate::compiled_views::models::{ModelInputHandles, SchemaHandles}; +use crate::resource_views::models::AuditProjections; +use crate::resource_views::models::FunctionProjections; +use crate::resource_views::models::SeedHandles; +use crate::resource_views::models::SourceProjections; + +impl Debug for ModelInputHandles { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("ModelInputHandles") + .finish_non_exhaustive() + } +} + +impl Debug for SchemaHandles { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("SchemaHandles") + .finish_non_exhaustive() + } +} + +impl Debug for CompiledProjections { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("CompiledProjections") + .finish_non_exhaustive() + } +} + +impl Debug for AnalysisProjections { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("AnalysisProjections") + .finish_non_exhaustive() + } +} + +impl Debug for SourceProjections { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("SourceProjections") + .finish_non_exhaustive() + } +} + +impl Debug for FunctionProjections { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("FunctionProjections") + .finish_non_exhaustive() + } +} + +impl Debug for AuditProjections { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("AuditProjections") + .finish_non_exhaustive() + } +} + +impl Debug for SeedHandles { + fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { + formatter + .debug_struct("SeedHandles") + .finish_non_exhaustive() + } +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_reads.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_reads.rs new file mode 100644 index 0000000000..d9f4098d68 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_reads.rs @@ -0,0 +1,25 @@ +//! A compiled project's resource collections, read from its native view. + +use pyo3::prelude::{Bound, PyAny, PyResult}; + +use crate::compiled_views::models::NativeProjectView; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; + +pub(crate) const MODELS_FIELD: &str = "models"; +pub(crate) const SOURCES_FIELD: &str = "sources"; +pub(crate) const SEEDS_FIELD: &str = "seeds"; +pub(crate) const FUNCTIONS_FIELD: &str = "functions"; +pub(crate) const AUDITS_FIELD: &str = "audits"; + +/// The native view behind a project object; any other object is a Python-built record, counted. +#[inline] +pub(crate) fn native_project<'a>( + project: &'a Bound<'_, PyAny>, +) -> PyResult> { + if let Ok(view) = project.cast::() { + return Ok(Some(view.get())); + } + count_stage(project.py(), COMPILED_VIEW_FALLBACKS, 1)?; + Ok(None) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs new file mode 100644 index 0000000000..eb60838df9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs @@ -0,0 +1,265 @@ +//! A compiled project, read from the native compiled project that owns its models. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +use crate::graph::models::ResourceRow; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +const MODELS_FIELD: &str = "models"; +const RUN_ID_FIELD: &str = "run_id"; +const ENFORCE_EXPLICIT_FIELD: &str = "enforce_explicit_references"; +const KEY_ATTRIBUTE: &str = "key"; +const DEPS_ATTRIBUTE: &str = "deps"; +pub(crate) const TAGS_ATTRIBUTE: &str = "tags"; +pub(crate) const SCHEMA_ENTRY_ATTRIBUTE: &str = "schema_entry"; +const RESOURCE_TYPE_ATTRIBUTE: &str = "resource_type"; +const NAME_ATTRIBUTE: &str = "name"; + +/// Project inputs held as Python objects: user code, discovery or an integration owns them. +#[derive(Debug)] +pub(crate) struct BoundaryInputs { + /// The resolved connection, adapter integration values. + pub(crate) effective_connection: Py, + /// Project variables, user values. + pub(crate) effective_vars: Py, + /// The native binding catalog. + pub(crate) binding_catalog: Py, + /// The adapter's lexical syntax. + pub(crate) sql_lexical_syntax: Py, + /// The compile cache directory path. + pub(crate) compile_cache_dir: Py, + /// Discovered loader functions, user callables. + pub(crate) loader_functions: Py, + /// Task/asset/check source locations, without retaining additional Python callables. + pub(crate) python_execution_inputs: BTreeMap, + /// Discovered Python hooks, user callables. + pub(crate) hook_functions: Py, + /// Discovered SQL hook files. + pub(crate) sql_hook_files: Py, + /// Discovered custom materializations. + pub(crate) materialization_files: Py, + /// Public enums, user declaration values. + pub(crate) public_enums: Py, + /// Public constants, user declaration values. + pub(crate) public_constants: Py, + /// Loaded macro modules, user code. + pub(crate) loaded_macros: Py, + /// The external reference resolver integration, or `None`. + pub(crate) external_sql_reference_resolver: Py, + /// The native analysis session, or `None`. + pub(crate) native_session: Py, +} + +/// Compiler-derived project parts still held as Python objects until they are owner facts. +#[derive(Debug)] +pub(crate) struct CompilerHandles { + /// Effective settings. + pub(crate) settings: Py, + /// Effective scenario config. + pub(crate) scenario: Py, + /// Compiled sources. + pub(crate) sources: Py, + /// Compiled seeds. + pub(crate) seeds: Py, + /// Compiled SQL functions. + pub(crate) functions: Py, + /// Compiled audits. + pub(crate) audits: Py, + /// Compiled SQL tests. + pub(crate) sql_tests: Py, + /// Compiled SQL scenarios. + pub(crate) sql_scenarios: Py, + /// Compiler diagnostics. + pub(crate) diagnostics: Py, + /// Literal relations no resource matched. + pub(crate) unmatched_literal_sql_relations: Py, + /// The declaration scope index. + pub(crate) scope_index: Py, +} + +/// The project-level values of one compiled project view. +#[derive(Debug)] +pub(crate) struct ProjectFields { + pub(crate) run_id: String, + pub(crate) effective_target_name: Option, + pub(crate) effective_target_database: Option, + pub(crate) effective_target_schema: Option, + pub(crate) sql_analysis_dialect: Option, + pub(crate) enforce_explicit_references: bool, + pub(crate) models: Py, + pub(crate) boundary: BoundaryInputs, + pub(crate) compiler: CompilerHandles, +} + +impl ProjectFields { + /// The fields named in `fields`, each required. + pub(crate) fn from_fields(fields: &Bound<'_, PyDict>) -> PyResult { + Ok(Self { + run_id: required(fields, RUN_ID_FIELD)?.extract()?, + effective_target_name: required(fields, "effective_target_name")?.extract()?, + effective_target_database: required(fields, "effective_target_database")?.extract()?, + effective_target_schema: required(fields, "effective_target_schema")?.extract()?, + sql_analysis_dialect: required(fields, "sql_analysis_dialect")?.extract()?, + enforce_explicit_references: required(fields, ENFORCE_EXPLICIT_FIELD)?.extract()?, + models: required(fields, MODELS_FIELD)? + .downcast_into::()? + .unbind(), + boundary: BoundaryInputs { + effective_connection: required(fields, "effective_connection")?.unbind(), + effective_vars: required(fields, "effective_vars")?.unbind(), + binding_catalog: required(fields, "binding_catalog")?.unbind(), + sql_lexical_syntax: required(fields, "sql_lexical_syntax")?.unbind(), + compile_cache_dir: required(fields, "compile_cache_dir")?.unbind(), + loader_functions: required(fields, "loader_functions")?.unbind(), + python_execution_inputs: required(fields, "python_execution_inputs")?.extract()?, + hook_functions: required(fields, "hook_functions")?.unbind(), + sql_hook_files: required(fields, "sql_hook_files")?.unbind(), + materialization_files: required(fields, "materialization_files")?.unbind(), + public_enums: required(fields, "public_enums")?.unbind(), + public_constants: required(fields, "public_constants")?.unbind(), + loaded_macros: required(fields, "loaded_macros")?.unbind(), + external_sql_reference_resolver: required( + fields, + "external_sql_reference_resolver", + )? + .unbind(), + native_session: required(fields, "native_session")?.unbind(), + }, + compiler: CompilerHandles { + settings: required(fields, "settings")?.unbind(), + scenario: required(fields, "scenario")?.unbind(), + sources: required(fields, "sources")?.unbind(), + seeds: required(fields, "seeds")?.unbind(), + functions: required(fields, "functions")?.unbind(), + audits: required(fields, "audits")?.unbind(), + sql_tests: required(fields, "sql_tests")?.unbind(), + sql_scenarios: required(fields, "sql_scenarios")?.unbind(), + diagnostics: required(fields, "diagnostics")?.unbind(), + unmatched_literal_sql_relations: required( + fields, + "unmatched_literal_sql_relations", + )? + .unbind(), + scope_index: required(fields, "scope_index")?.unbind(), + }, + }) + } + + /// These fields with `changes` applied. + pub(crate) fn changed(&self, py: Python<'_>, changes: &Bound<'_, PyDict>) -> PyResult { + let current: Bound<'_, PyDict> = self.as_fields(py)?; + for (name, value) in changes.iter() { + if !current.contains(&name)? { + return Err(PyTypeError::new_err(format!( + "unknown project field {name}" + ))); + } + current.set_item(name, value)?; + } + Self::from_fields(¤t) + } + + /// Every field by name. + pub(crate) fn as_fields<'py>(&self, py: Python<'py>) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item(RUN_ID_FIELD, &self.run_id)?; + fields.set_item("effective_target_name", &self.effective_target_name)?; + fields.set_item("effective_target_database", &self.effective_target_database)?; + fields.set_item("effective_target_schema", &self.effective_target_schema)?; + fields.set_item("sql_analysis_dialect", &self.sql_analysis_dialect)?; + fields.set_item(ENFORCE_EXPLICIT_FIELD, self.enforce_explicit_references)?; + fields.set_item(MODELS_FIELD, self.models.bind(py))?; + fields.set_item( + "effective_connection", + self.boundary.effective_connection.bind(py), + )?; + fields.set_item("effective_vars", self.boundary.effective_vars.bind(py))?; + fields.set_item("binding_catalog", self.boundary.binding_catalog.bind(py))?; + fields.set_item( + "sql_lexical_syntax", + self.boundary.sql_lexical_syntax.bind(py), + )?; + fields.set_item( + "compile_cache_dir", + self.boundary.compile_cache_dir.bind(py), + )?; + fields.set_item("loader_functions", self.boundary.loader_functions.bind(py))?; + fields.set_item( + "python_execution_inputs", + &self.boundary.python_execution_inputs, + )?; + fields.set_item("hook_functions", self.boundary.hook_functions.bind(py))?; + fields.set_item("sql_hook_files", self.boundary.sql_hook_files.bind(py))?; + fields.set_item( + "materialization_files", + self.boundary.materialization_files.bind(py), + )?; + fields.set_item("public_enums", self.boundary.public_enums.bind(py))?; + fields.set_item("public_constants", self.boundary.public_constants.bind(py))?; + fields.set_item("loaded_macros", self.boundary.loaded_macros.bind(py))?; + fields.set_item( + "external_sql_reference_resolver", + self.boundary.external_sql_reference_resolver.bind(py), + )?; + fields.set_item("native_session", self.boundary.native_session.bind(py))?; + fields.set_item("settings", self.compiler.settings.bind(py))?; + fields.set_item("scenario", self.compiler.scenario.bind(py))?; + fields.set_item("sources", self.compiler.sources.bind(py))?; + fields.set_item("seeds", self.compiler.seeds.bind(py))?; + fields.set_item("functions", self.compiler.functions.bind(py))?; + fields.set_item("audits", self.compiler.audits.bind(py))?; + fields.set_item("sql_tests", self.compiler.sql_tests.bind(py))?; + fields.set_item("sql_scenarios", self.compiler.sql_scenarios.bind(py))?; + fields.set_item("diagnostics", self.compiler.diagnostics.bind(py))?; + fields.set_item( + "unmatched_literal_sql_relations", + self.compiler.unmatched_literal_sql_relations.bind(py), + )?; + fields.set_item("scope_index", self.compiler.scope_index.bind(py))?; + Ok(fields) + } +} + +fn required<'py>(fields: &Bound<'py, PyDict>, name: &str) -> PyResult> { + fields + .get_item(name)? + .ok_or_else(|| PyTypeError::new_err(format!("missing project field {name}"))) +} + +/// A source, seed or function's graph row, read from its compiled object. +pub(crate) fn resource_row( + resource: &Bound<'_, PyAny>, + tags: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let mut deps: Vec<(String, String)> = Vec::new(); + for key in native_field(resource, DEPS_ATTRIBUTE)?.try_iter()? { + deps.push(key_row(&key?)?); + } + let mut tag_names: Vec = Vec::new(); + if let Some(tags) = tags { + for tag in tags.try_iter()? { + tag_names.push(tag?.str()?.extract()?); + } + } + Ok(ResourceRow { + key: key_row(&native_field(resource, KEY_ATTRIBUTE)?)?, + deps, + tags: tag_names, + folder: None, + }) +} + +/// `(resource type, name)` of a `CompiledObjectKey`. +pub(crate) fn key_row(key: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + Ok(( + native_field(key, RESOURCE_TYPE_ATTRIBUTE)? + .str()? + .extract()?, + native_field(key, NAME_ATTRIBUTE)?.extract()?, + )) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs new file mode 100644 index 0000000000..ff33af9d6f --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs @@ -0,0 +1,251 @@ +//! Compiled records and views: projecting native views as records, and recording later stages. + +use crate::compiled_views::main::project::declaration_rows::declaration_rows; +use crate::compiled_views::main::project::plain_typed_value::plain_typed_value; +use crate::compiled_views::main::project::record_model_analyses::record_model_analyses; +use crate::compiled_views::main::project::replace::replace; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple, PyType}; +use pyo3::{IntoPyObjectExt, pyfunction}; +use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; + +use crate::compiled_project::main::entry_schema_fact::entry_schema_fact; +use crate::compiled_project::main::model_facts::model_facts; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::NativeProjectView; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::record_spec::record_spec; +use sqlbuild_python_base::records::types::RecordId; + +pub(crate) const DATACLASSES_MODULE: &str = "dataclasses"; +pub(crate) const DECIMAL_KIND: &str = "decimal"; +pub(crate) const COLLECTION_KINDS: [&str; 2] = ["list", "set"]; +pub(crate) const SCALAR_KINDS: [&str; 5] = ["string", "integer", "boolean", "float", "null"]; + +/// `value` when it already is a `class` record, else `class(**{field: getattr(value, field)})`. +pub(crate) fn record<'py>( + value: &Bound<'py, PyAny>, + class: RecordId, +) -> PyResult> { + let py: Python<'py> = value.py(); + let class: Bound<'py, PyAny> = record_class(py, class)?; + if value.is_instance(&class)? { + return Ok(value.clone()); + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + for field in record_spec(class.cast::()?)?.fields { + fields.set_item(field.name, native_field(value, field.name)?)?; + } + class.call((), Some(&fields)) +} + +/// Each item of `fields[name]` as a `class` record. +fn records_of<'py>(fields: &Bound<'py, PyDict>, (name, class): (&str, RecordId)) -> PyResult<()> { + let py: Python<'py> = fields.py(); + let items: Vec> = fields + .as_any() + .get_item(name)? + .try_iter()? + .map(|item| record(&item?, class)) + .collect::>()?; + fields.set_item(name, PyTuple::new(py, items)?) +} + +/// The project as a `CompiledProject`, projected from its owner when it is a native view. +#[pyfunction] +#[pyo3(signature = (project, *, include_sql_expansions=true))] +pub(crate) fn compiled_project_record<'py>( + project: &Bound<'py, PyAny>, + include_sql_expansions: bool, +) -> PyResult> { + let py: Python<'py> = project.py(); + let class: Bound<'py, PyAny> = record_class(py, RecordId::CompiledProject)?; + if project.is_instance(&class)? { + if include_sql_expansions { + return Ok(project.clone()); + } + return replace(project, &[("sql_expansions", PyDict::new(py).into_any())]); + } + let view: PyRef<'py, NativeProjectView> = project.extract()?; + let fields: Bound<'py, PyDict> = view.fields(py, include_sql_expansions)?; + drop(view); + for pair in [ + ("seeds", RecordId::CompiledSeed), + ("sources", RecordId::CompiledSource), + ("functions", RecordId::CompiledFunction), + ("audits", RecordId::CompiledAudit), + ] { + records_of(&fields, pair)?; + } + new_record(py, RecordId::CompiledProject, (), Some(&fields)) +} + +/// `replace(value, **changes)` for a record, else the native view's `with_changes(**changes)`. +pub(crate) fn with_changes<'py>( + value: &Bound<'py, PyAny>, + class: RecordId, + changes: Option>, +) -> PyResult> { + let py: Python<'py> = value.py(); + let changes: Bound<'py, PyDict> = changes.unwrap_or_else(|| PyDict::new(py)); + if value.is_instance(&record_class(py, class)?)? { + return imported(py, DATACLASSES_MODULE, "replace")?.call((value,), Some(&changes)); + } + value.call_method("with_changes", (), Some(&changes)) +} + +/// An enum declaration as the native project row. +pub(crate) fn enum_row<'py>(declaration: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = declaration.py(); + let members: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = + native_field(declaration, "members")? + .try_iter()? + .map(|member| { + let member: Bound<'py, PyAny> = member?; + Ok(( + native_field(&member, "name")?, + native_field(&member, "value")?, + )) + }) + .collect::>()?; + PyTuple::new( + py, + [ + native_field(declaration, "name")?, + native_field(declaration, "relative_path")?.call_method0("as_posix")?, + members.into_bound_py_any(py)?, + py.None().into_bound(py), + py.None().into_bound(py), + py.None().into_bound(py), + ], + ) +} + +/// A constant declaration as the native project row, its value made JSON-safe. +pub(crate) fn constant_row<'py>(declaration: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = declaration.py(); + let render_as: Bound<'py, PyAny> = native_field(declaration, "render_as")?; + PyTuple::new( + py, + [ + native_field(declaration, "name")?, + native_field(declaration, "relative_path")?.call_method0("as_posix")?, + PyList::empty(py).into_any(), + plain_typed_value(&native_field(declaration, "value")?)?, + native_field(declaration, "logical_type")?.getattr("display_name")?, + if render_as.is_none() { + render_as + } else { + native_field(&render_as, "value")? + }, + ], + ) +} + +/// `(name, type)` of each dynamic family; `casefold` keys the family by its folded name. +pub(crate) fn family_types<'py>( + families: &Bound<'py, PyAny>, + casefold: bool, +) -> PyResult)>> { + families + .try_iter()? + .map(|family| { + let family: Bound<'py, PyAny> = family?; + let name: Bound<'py, PyAny> = native_field(&family, "name")?; + let name: String = if casefold { + name.call_method0("casefold")?.extract()? + } else { + name.extract()? + }; + Ok((name, native_field(&family, "inferred_type")?.extract()?)) + }) + .collect() +} + +/// A native project for a `CompiledProject` built without the compile-input stage. +pub(crate) fn hand_built_project<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let native: Bound<'py, NativeCompiledProject> = + Bound::new(py, NativeCompiledProject::default())?; + let models: Bound<'py, PyAny> = native_field(project, "models")?; + for model in models.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let mut facts: CompiledModelFacts = model_facts(model_row(&model)?.as_any().extract()?)?; + facts.schema = entry_schema_fact( + &native_field(&model, "schema_entry")?, + &native_field(&model, "relative_path")?, + )?; + native.borrow_mut().retain(facts); + } + let _ = record_model_analyses(&native, &models)?; + Ok(native) +} + +fn model_row<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + let relative_path: Bound<'py, PyAny> = native_field(model, "relative_path")?; + let config: Bound<'py, PyAny> = native_field(model, "config")?; + let references: Vec> = native_field(model, "references")? + .try_iter()? + .map(|reference| { + let reference: Bound<'py, PyAny> = reference?; + PyTuple::new( + py, + [ + native_field(&reference, "ref_kind")?.str()?.into_any(), + native_field(&reference, "ref_name")?, + native_field(&reference, "ref_package")?, + ], + ) + }) + .collect::>()?; + let (enums, constants) = declaration_rows( + &native_field(model, "enum_declarations")?, + &native_field(model, "constant_declarations")?, + )?; + PyTuple::new( + py, + [ + PyTuple::new( + py, + [ + native_field(model, "name")?, + relative_path.call_method0("as_posix")?, + native_field(model, "query_sql")?, + native_field(model, "authored_query_sql")?, + native_field(model, "authored_sql")?, + ], + )? + .into_any(), + PyTuple::new( + py, + [ + native_field(&config, "values")?, + PyList::new( + py, + native_field(&config, "model_header_keys")? + .try_iter()? + .collect::>>()?, + )? + .into_any(), + native_field(&config, "layer_schema")?, + ], + )? + .into_any(), + PyList::new(py, references)?.into_any(), + PyList::new( + py, + native_field(model, "enum_columns")? + .try_iter()? + .collect::>>()?, + )? + .into_any(), + PyTuple::new(py, [enums.into_any(), constants.into_any()])?.into_any(), + ], + ) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/constants.rs b/crates/sqlbuild-python-project/src/compiled_views/constants.rs new file mode 100644 index 0000000000..05bd22c3c7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/constants.rs @@ -0,0 +1,80 @@ +//! Public constants of compiled views. + +use crate::compiled_views::_helpers::project::getters::read_native_audit_input; +use crate::compiled_views::_helpers::project::getters::read_native_compiled_audit; +use crate::compiled_views::_helpers::project::getters::read_native_compiled_function; +use crate::compiled_views::_helpers::project::getters::read_native_compiled_model; +use crate::compiled_views::_helpers::project::getters::read_native_compiled_seed; +use crate::compiled_views::_helpers::project::getters::read_native_compiled_source; +use crate::compiled_views::_helpers::project::getters::read_native_function_input; +use crate::compiled_views::_helpers::project::getters::read_native_inputs_view; +use crate::compiled_views::_helpers::project::getters::read_native_model_input; +use crate::compiled_views::_helpers::project::getters::read_native_project_view; +use crate::compiled_views::_helpers::project::getters::read_native_seed_input; +use crate::compiled_views::_helpers::project::getters::read_native_source_input; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeInputsView; +use crate::compiled_views::models::NativeModelInput; +use crate::compiled_views::models::NativeProjectView; +use crate::resource_views::models::NativeAuditInput; +use crate::resource_views::models::NativeCompiledAudit; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; +use crate::resource_views::models::NativeFunctionInput; +use crate::resource_views::models::NativeSeedInput; +use crate::resource_views::models::NativeSourceInput; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_native_audit_input, + }, + GetterEntry { + class: class_of::, + read: read_native_compiled_audit, + }, + GetterEntry { + class: class_of::, + read: read_native_compiled_model, + }, + GetterEntry { + class: class_of::, + read: read_native_compiled_function, + }, + GetterEntry { + class: class_of::, + read: read_native_function_input, + }, + GetterEntry { + class: class_of::, + read: read_native_inputs_view, + }, + GetterEntry { + class: class_of::, + read: read_native_model_input, + }, + GetterEntry { + class: class_of::, + read: read_native_project_view, + }, + GetterEntry { + class: class_of::, + read: read_native_compiled_seed, + }, + GetterEntry { + class: class_of::, + read: read_native_seed_input, + }, + GetterEntry { + class: class_of::, + read: read_native_compiled_source, + }, + GetterEntry { + class: class_of::, + read: read_native_source_input, + }, +]; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/audit_key_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/audit_key_rows.rs new file mode 100644 index 0000000000..944f49032e --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/audit_key_rows.rs @@ -0,0 +1,25 @@ +//! Each audit spec's `(definition name, column argument when it is text)`. + +use crate::compiled_views::_helpers::compiled_models::relation_keys::UNIQUE_AUDIT_COLUMN_ARGUMENT; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Each audit spec's `(definition name, column argument when it is text)`. +pub fn audit_key_rows(audits: &Bound<'_, PyAny>) -> PyResult)>> { + let mut rows: Vec<(String, Option)> = Vec::new(); + for audit in audits.try_iter()? { + let audit: Bound<'_, PyAny> = audit?; + let argument: Option> = native_field(&audit, "arguments")? + .downcast::()? + .get_item(UNIQUE_AUDIT_COLUMN_ARGUMENT)?; + rows.push(( + native_field(&audit, "definition_name")?.extract()?, + argument + .filter(pyo3::types::PyAnyMethods::is_instance_of::) + .map(|value| value.extract()) + .transpose()?, + )); + } + Ok(rows) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/binding_validated.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/binding_validated.rs new file mode 100644 index 0000000000..312064be6d --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/binding_validated.rs @@ -0,0 +1,13 @@ +//! Whether the native session validated the model's bindings. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Whether the native session validated the model's bindings. +pub fn binding_validated(model: &Bound<'_, PyAny>) -> PyResult { + if let Some(view) = native_model(model)? { + return view.binding_validated(model.py()); + } + native_field(model, "binding_validated")?.is_truthy() +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/config.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/config.rs new file mode 100644 index 0000000000..784eea98b3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/config.rs @@ -0,0 +1,14 @@ +//! The model's `CompileModelConfig`. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The model's `CompileModelConfig`. +pub fn config<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.config(py)?.into_bound(py)); + } + native_field(model, "config") +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/declared_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/declared_rows.rs new file mode 100644 index 0000000000..19274ea22b --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/declared_rows.rs @@ -0,0 +1,14 @@ +//! Each declared column of the attached schema as `(name, type)`, `None` without a schema. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use crate::compiled_views::types::DeclaredRow; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Each declared column of the attached schema as `(name, type)`, `None` without a schema. +pub fn declared_rows(model: &Bound<'_, PyAny>) -> PyResult>> { + if let Some(view) = native_model(model)? { + return view.schema_column_rows(model.py()); + } + native_field(model, "schema_column_rows")?.extract() +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/defaulted_destination.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/defaulted_destination.rs new file mode 100644 index 0000000000..b66d1e8d9a --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/defaulted_destination.rs @@ -0,0 +1,31 @@ +//! The destination with adapter defaults filled in; `render` renders its qualified name. + +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::DATABASE_KEY; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::SCHEMA_KEY; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_project::models::DestinationFact; + +/// The destination with adapter defaults filled in; `render` renders its qualified name. +pub fn defaulted_destination( + current: &DestinationFact, + defaults: (Option, Option), + render: &Bound<'_, PyAny>, +) -> PyResult { + let (default_schema, default_database) = defaults; + let schema: Option = current.schema.clone().or(default_schema); + let database: Option = current.database.clone().or(default_database); + let keywords: Bound<'_, PyDict> = PyDict::new(render.py()); + keywords.set_item(DATABASE_KEY, &database)?; + keywords.set_item(SCHEMA_KEY, &schema)?; + keywords.set_item("name", ¤t.name)?; + let qualified_name: Option = render.call((), Some(&keywords))?.extract()?; + Ok(DestinationFact { + database, + schema, + name: current.name.clone(), + qualified_name, + logical_schema: current.logical_schema.clone(), + logical_database: current.logical_database.clone(), + }) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dep_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dep_rows.rs new file mode 100644 index 0000000000..9347d09cf7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dep_rows.rs @@ -0,0 +1,13 @@ +//! `(resource type, name)` of each dependency. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// `(resource type, name)` of each dependency. +pub fn dep_rows(model: &Bound<'_, PyAny>) -> PyResult> { + if let Some(view) = native_model(model)? { + return view.dep_rows(model.py()); + } + native_field(model, "dep_rows")?.extract() +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/deps.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/deps.rs new file mode 100644 index 0000000000..f5262b70a1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/deps.rs @@ -0,0 +1,14 @@ +//! The model's dependency `CompiledObjectKey` tuple. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The model's dependency `CompiledObjectKey` tuple. +pub fn deps<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.deps(py)?.into_bound(py).into_any()); + } + native_field(model, "deps") +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/destination.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/destination.rs new file mode 100644 index 0000000000..e0813b1511 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/destination.rs @@ -0,0 +1,13 @@ +//! The model's destination `CompiledRelationLocation`. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The model's destination `CompiledRelationLocation`. +pub fn destination<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + if let Some(view) = native_model(model)? { + return Ok(view.destination(model.py())?.into_bound(model.py())); + } + native_field(model, "destination") +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dynamic_columns_declared.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dynamic_columns_declared.rs new file mode 100644 index 0000000000..71b6881793 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dynamic_columns_declared.rs @@ -0,0 +1,13 @@ +//! Whether the attached schema declares dynamic column families. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Whether the attached schema declares dynamic column families. +pub fn dynamic_columns_declared(model: &Bound<'_, PyAny>) -> PyResult { + if let Some(view) = native_model(model)? { + return view.dynamic_columns_declared(model.py()); + } + native_field(model, "has_dynamic_columns")?.is_truthy() +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/entry_relation_key_row.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/entry_relation_key_row.rs new file mode 100644 index 0000000000..6508f40189 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/entry_relation_key_row.rs @@ -0,0 +1,31 @@ +//! The key row of a seed, source or hand-built model from its config values and schema entry. + +use crate::compiled_project::main::python_column_facts::python_column_facts; +use crate::compiled_views::_helpers::compiled_models::relation_keys::declared_specs; +use crate::compiled_views::_helpers::compiled_models::relation_keys::relation_key_row; +use crate::compiled_views::main::compiled_models::audit_key_rows::audit_key_rows; +use crate::compiled_views::models::RelationKeyRow; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyDict; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; + +/// The key row of a seed, source or hand-built model from its config values and schema entry. +pub fn entry_relation_key_row( + kind: &str, + name: String, + values: &Bound<'_, PyDict>, + entry: &Bound<'_, PyAny>, +) -> PyResult { + if entry.is_none() { + return relation_key_row(kind, name, values, None); + } + let columns: Vec = + python_column_facts(&declared_specs(entry, "columns")?, None)?; + let audits: Bound<'_, PyAny> = declared_specs(entry, "audits")?; + relation_key_row( + kind, + name, + values, + Some((&columns, audit_key_rows(&audits)?)), + ) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/enum_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/enum_columns.rs new file mode 100644 index 0000000000..9d537e3e7c --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/enum_columns.rs @@ -0,0 +1,14 @@ +//! The model's enum columns by name. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The model's enum columns by name. +pub fn enum_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.enum_columns(py)?.into_bound(py).into_any()); + } + native_field(model, "enum_columns") +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_columns.rs new file mode 100644 index 0000000000..43c632f13a --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_columns.rs @@ -0,0 +1,14 @@ +//! The model's `InferredColumn` tuple, or `None`. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The model's `InferredColumn` tuple, or `None`. +pub fn inferred_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.inferred_columns(py)?.into_bound(py)); + } + native_field(model, "inferred_columns") +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_names.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_names.rs new file mode 100644 index 0000000000..72b1871db2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_names.rs @@ -0,0 +1,10 @@ +//! The names of the inferred output columns, `None` when the shape is unknown. + +use crate::compiled_views::_helpers::compiled_models::model_reads::column_names; +use crate::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// The names of the inferred output columns, `None` when the shape is unknown. +pub fn inferred_names(model: &Bound<'_, PyAny>) -> PyResult>> { + Ok(inferred_rows(model)?.map(column_names)) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_rows.rs new file mode 100644 index 0000000000..1269f953f2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_rows.rs @@ -0,0 +1,24 @@ +//! Each inferred output column as `(name, type, nullability)`, `None` when none were inferred. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_project::types::InferredRow; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Each inferred output column as `(name, type, nullability)`, `None` when none were inferred. +pub fn inferred_rows(model: &Bound<'_, PyAny>) -> PyResult>> { + if let Some(view) = native_model(model)? { + return view.inferred_rows(model.py()); + } + let rows: Bound<'_, PyAny> = native_field(model, "inferred_rows")?; + if rows.is_none() { + return Ok(None); + } + let mut inferred: Vec = Vec::new(); + for row in rows.try_iter()? { + let (name, column_type, nullability): (String, Option, Bound<'_, PyAny>) = + row?.extract()?; + inferred.push((name, column_type, nullability.str()?.extract()?)); + } + Ok(Some(inferred)) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_facts.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_facts.rs new file mode 100644 index 0000000000..d9148405c9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_facts.rs @@ -0,0 +1,52 @@ +//! Each lineage output with its transform kind and confidence values, `None` without lineage. + +use crate::compiled_views::_helpers::compiled_models::model_reads::CONFIDENCE_VALUES; +use crate::compiled_views::_helpers::compiled_models::model_reads::TRANSFORM_VALUES; +use crate::compiled_views::_helpers::compiled_models::model_reads::coded; +use crate::compiled_views::main::compiled_models::native_model::native_model; +use crate::compiled_views::models::LineageFactRow; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Each lineage output with its transform kind and confidence values, `None` without lineage. +pub fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult>> { + if let Some(view) = native_model(model)? { + let Some(rows) = view.lineage_rows(model.py())? else { + return Ok(None); + }; + let mut facts: Vec = Vec::with_capacity(rows.len()); + for row in rows { + facts.push(LineageFactRow { + output_column: row.output_column, + transform: coded(&TRANSFORM_VALUES, row.transform_code)?, + confidence: coded(&CONFIDENCE_VALUES, row.confidence_code)?, + sources: row.sources, + }); + } + return Ok(Some(facts)); + } + let columns: Bound<'_, PyAny> = native_field(model, "fast_lineage_columns")?; + if columns.is_none() { + return Ok(None); + } + let mut facts: Vec = Vec::new(); + for output in columns.try_iter()? { + let output: Bound<'_, PyAny> = output?; + let mut sources: Vec<(String, String, String)> = Vec::new(); + for source in native_field(&output, "upstream_columns")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + sources.push(( + native_field(&source, "resource_type")?.extract()?, + native_field(&source, "resource_name")?.extract()?, + native_field(&source, "column_name")?.extract()?, + )); + } + facts.push(LineageFactRow { + output_column: native_field(&output, "output_column")?.extract()?, + transform: native_field(&output, "transform_kind")?.str()?.extract()?, + confidence: native_field(&output, "confidence")?.str()?.extract()?, + sources, + }); + } + Ok(Some(facts)) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_inputs.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_inputs.rs new file mode 100644 index 0000000000..ea19e402b3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/lineage_inputs.rs @@ -0,0 +1,31 @@ +//! Each lineage output with the `(resource, column)` pairs it reads; empty without lineage. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use crate::compiled_views::types::LineageInput; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Each lineage output with the `(resource, column)` pairs it reads; empty without lineage. +pub fn lineage_inputs(model: &Bound<'_, PyAny>) -> PyResult> { + if let Some(view) = native_model(model)? { + return Ok(view.lineage_inputs(model.py())?.unwrap_or_default()); + } + let columns: Bound<'_, PyAny> = native_field(model, "fast_lineage_columns")?; + if !columns.is_truthy()? { + return Ok(Vec::new()); + } + let mut lineage: Vec = Vec::new(); + for output in columns.try_iter()? { + let output: Bound<'_, PyAny> = output?; + let mut upstream: Vec<(String, String)> = Vec::new(); + for source in native_field(&output, "upstream_columns")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + upstream.push(( + native_field(&source, "resource_name")?.extract()?, + native_field(&source, "column_name")?.extract()?, + )); + } + lineage.push((native_field(&output, "output_column")?.extract()?, upstream)); + } + Ok(lineage) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/mod.rs new file mode 100644 index 0000000000..3cc771aff9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/mod.rs @@ -0,0 +1,37 @@ +pub mod audit_key_rows; +pub mod binding_validated; +pub mod config; +pub mod declared_rows; +pub mod defaulted_destination; +pub mod dep_rows; +pub mod deps; +pub mod destination; +pub mod dynamic_columns_declared; +pub mod entry_relation_key_row; +pub mod enum_columns; +pub mod inferred_columns; +pub mod inferred_names; +pub mod inferred_rows; +pub mod lineage_facts; +pub mod lineage_inputs; +pub mod model_graph_row; +pub mod model_key; +pub mod model_key_row; +pub mod model_name; +pub mod name_object; +pub mod native_model; +pub mod object_key; +pub mod one_row_seed_names; +pub mod output_proven; +pub mod project_relation_keys; +pub mod proven_inputs; +pub mod query_sql; +pub mod references; +pub mod register_relation_keys; +pub mod relation_keys; +pub mod relation_location; +pub mod relative_path; +pub mod schema_entry; +pub mod source_key_input; +pub mod star_flags; +pub mod unchecked_columns; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_graph_row.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_graph_row.rs new file mode 100644 index 0000000000..56de1cd5c2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_graph_row.rs @@ -0,0 +1,37 @@ +//! The model's project graph row: its key, dependencies, tags and folder. + +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::CompiledRecord; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::MODEL_RESOURCE; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::TAGS_KEY; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::VALUES_ATTRIBUTE; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::compiled_of; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::model_name; +use crate::compiled_views::models::NativeCompiledModel; +use crate::graph::models::ResourceRow; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The model's project graph row: its key, dependencies, tags and folder. +pub fn model_graph_row(model: &NativeCompiledModel, py: Python<'_>) -> PyResult { + model.read(py, |record, py| { + let compiled: &CompiledRecord = compiled_of(record)?; + let values: Bound<'_, PyAny> = native_field(record.host.config.bind(py), VALUES_ATTRIBUTE)?; + let mut tags: Vec = Vec::new(); + if let Some(value) = values.downcast::()?.get_item(TAGS_KEY)? + && (value.is_instance_of::() + || value.is_instance_of::()) + { + for tag in value.try_iter()? { + tags.push(tag?.str()?.extract()?); + } + } + let folder: String = record.host.model_file.get().folder(py)?; + Ok(ResourceRow { + key: (MODEL_RESOURCE.to_owned(), model_name(record)), + deps: compiled.deps.clone(), + tags, + folder: Some(folder), + }) + }) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key.rs new file mode 100644 index 0000000000..9cfd7328bd --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key.rs @@ -0,0 +1,13 @@ +//! The model's `CompiledObjectKey`. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The model's `CompiledObjectKey`. +pub fn model_key<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + if let Some(view) = native_model(model)? { + return Ok(view.key(model.py())?.into_bound(model.py())); + } + native_field(model, "key") +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key_row.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key_row.rs new file mode 100644 index 0000000000..a523cb302e --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key_row.rs @@ -0,0 +1,22 @@ +//! The declared key facts of a model input or compiled model. + +use crate::compiled_views::_helpers::model_inputs::model_input_view::model_relation_key_row; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeModelInput; +use crate::compiled_views::models::RelationKeyRow; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The declared key facts of a model input or compiled model. +pub fn model_key_row(model: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = model.py(); + if let Ok(input) = model.downcast::() { + return input.get().read_with_owner(py, |owner, record| { + model_relation_key_row(owner, record, py) + }); + } + if let Ok(compiled) = model.downcast::() { + return compiled.get().relation_key_row(py); + } + native_field(model, "relation_key_row")?.extract() +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_name.rs new file mode 100644 index 0000000000..2519953905 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_name.rs @@ -0,0 +1,13 @@ +//! The model's name. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The model's name. +pub fn model_name(model: &Bound<'_, PyAny>) -> PyResult { + if let Some(view) = native_model(model)? { + return view.name(model.py()); + } + native_field(model, "name")?.extract() +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/name_object.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/name_object.rs new file mode 100644 index 0000000000..e36c20de58 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/name_object.rs @@ -0,0 +1,14 @@ +//! The model's name as a Python string. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The model's name as a Python string. +pub fn name_object<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + if let Some(view) = native_model(model)? { + let name: String = view.name(model.py())?; + return Ok(pyo3::types::PyString::new(model.py(), &name).into_any()); + } + native_field(model, "name") +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/native_model.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/native_model.rs new file mode 100644 index 0000000000..53d5b3464c --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/native_model.rs @@ -0,0 +1,16 @@ +//! The native view behind a compiled model object. + +use crate::compiled_views::models::NativeCompiledModel; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; + +/// The native view behind a model object; any other object is a Python-built record, counted. +#[inline] +pub fn native_model<'a>(model: &'a Bound<'_, PyAny>) -> PyResult> { + if let Ok(view) = model.cast::() { + return Ok(Some(view.get())); + } + count_stage(model.py(), COMPILED_VIEW_FALLBACKS, 1)?; + Ok(None) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/object_key.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/object_key.rs new file mode 100644 index 0000000000..a3774bdfea --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/object_key.rs @@ -0,0 +1,17 @@ +//! Object key. + +use crate::compiled_views::models::CompiledViewClasses; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; + +/// Object key. +pub fn object_key( + classes: &CompiledViewClasses<'_>, + kind: &str, + name: &str, +) -> PyResult> { + let fields: Bound<'_, PyDict> = PyDict::new(classes.object_key.py()); + fields.set_item("resource_type", kind)?; + fields.set_item("name", name)?; + Ok(classes.object_key.call((), Some(&fields))?.unbind()) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/one_row_seed_names.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/one_row_seed_names.rs new file mode 100644 index 0000000000..1d22f5bab5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/one_row_seed_names.rs @@ -0,0 +1,32 @@ +//! Names of the seeds whose CSV holds at most one data row, from `seed_file` and `schema_entry`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_discovery::project_validation::main::at_most_one_csv_data_row::at_most_one_csv_data_row; +use sqlbuild_discovery::project_validation::models::CsvSettings; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::main::project::seed_csv_settings::seed_csv_settings; +use std::path::PathBuf; + +/// Names of the seeds whose CSV holds at most one data row, from `seed_file` and `schema_entry`. +pub fn one_row_seed_names(seeds: &[Bound<'_, PyAny>]) -> PyResult> { + let mut candidates: Vec<(String, PathBuf, CsvSettings)> = Vec::with_capacity(seeds.len()); + for seed in seeds { + let entry: Bound<'_, PyAny> = native_field(seed, "schema_entry")?; + candidates.push(( + native_field(&entry, "name")?.extract()?, + native_field(&native_field(seed, "seed_file")?, "file_path")?.extract()?, + seed_csv_settings(&entry)?, + )); + } + let py: Python<'_> = match seeds.first() { + Some(seed) => seed.py(), + None => return Ok(Vec::new()), + }; + Ok(py.detach(|| { + candidates + .into_iter() + .filter(|(_, path, settings)| at_most_one_csv_data_row(path, settings)) + .map(|(name, _, _)| name) + .collect() + })) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/output_proven.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/output_proven.rs new file mode 100644 index 0000000000..3fb73f9ca2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/output_proven.rs @@ -0,0 +1,14 @@ +//! Whether the model's dynamic pivot proof proved its output columns. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Whether the model's dynamic pivot proof proved its output columns. +pub fn output_proven(model: &Bound<'_, PyAny>) -> PyResult { + if let Some(view) = native_model(model)? { + return view.output_proven(model.py()); + } + let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?; + Ok(!proof.is_none() && native_field(&proof, "output_proven")?.is_truthy()?) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/project_relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/project_relation_keys.rs new file mode 100644 index 0000000000..6959869ae0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/project_relation_keys.rs @@ -0,0 +1,41 @@ +//! Declared keys of a compiled project's models, seeds and sources. + +use crate::compiled_views::main::compiled_models::model_key_row::model_key_row; +use crate::compiled_views::main::compiled_models::one_row_seed_names::one_row_seed_names; +use crate::compiled_views::main::compiled_models::relation_keys::relation_keys; +use crate::compiled_views::main::compiled_models::source_key_input::source_key_input; +use crate::compiled_views::models::RelationKeyRow; +use crate::compiled_views::types::RelationKeys; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Declared keys of a compiled project's models, seeds and sources. +pub fn project_relation_keys(project: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = project.py(); + let mut rows: Vec = Vec::new(); + for model in native_field(project, "models")?.try_iter()? { + rows.push(model_key_row(&model?)?); + } + let compiled_seeds: Vec> = native_field(project, "seeds")? + .try_iter()? + .collect::>()?; + let one_row_seeds: Vec = one_row_seed_names(&compiled_seeds)?; + let mut seeds: Vec> = Vec::new(); + for seed in &compiled_seeds { + seeds.push(native_field(seed, "schema_entry")?); + } + let mut sources: Vec> = Vec::new(); + for source in native_field(project, "sources")?.try_iter()? { + sources.push(source_key_input(&source?)?); + } + relation_keys( + py, + rows, + ( + PyTuple::new(py, seeds)?.as_any(), + PyTuple::new(py, sources)?.as_any(), + ), + &one_row_seeds, + ) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/proven_inputs.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/proven_inputs.rs new file mode 100644 index 0000000000..2fc54f0561 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/proven_inputs.rs @@ -0,0 +1,17 @@ +//! The input relations of a dynamic pivot proof that proved the model's outputs, else `None`. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The input relations of a dynamic pivot proof that proved the model's outputs, else `None`. +pub fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult>> { + if let Some(view) = native_model(model)? { + return view.proven_inputs(model.py()); + } + let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?; + if proof.is_none() || !native_field(&proof, "output_proven")?.is_truthy()? { + return Ok(None); + } + native_field(&proof, "input_relations")?.extract().map(Some) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/query_sql.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/query_sql.rs new file mode 100644 index 0000000000..e77775791a --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/query_sql.rs @@ -0,0 +1,13 @@ +//! The model's compiled query SQL. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The model's compiled query SQL. +pub fn query_sql<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + if let Some(view) = native_model(model)? { + return Ok(view.query_sql(model.py())?.into_any()); + } + native_field(model, "query_sql") +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/references.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/references.rs new file mode 100644 index 0000000000..746c48cac4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/references.rs @@ -0,0 +1,14 @@ +//! The model's `CompileSqlReference` tuple. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The model's `CompileSqlReference` tuple. +pub fn references<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.references(py)?.into_bound(py).into_any()); + } + native_field(model, "references") +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/register_relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/register_relation_keys.rs new file mode 100644 index 0000000000..095190dea8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/register_relation_keys.rs @@ -0,0 +1,12 @@ +//! Register the relation keys functions and classes. + +use crate::compiled_views::_helpers::compiled_models::relation_keys::compiled_relation_keys; +use crate::compiled_views::_helpers::compiled_models::relation_keys::declared_relation_keys; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the relation keys functions and classes. +pub fn register_relation_keys(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(declared_relation_keys, module)?)?; + module.add_function(wrap_pyfunction!(compiled_relation_keys, module)?) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_keys.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_keys.rs new file mode 100644 index 0000000000..0ef40b6b31 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_keys.rs @@ -0,0 +1,47 @@ +//! Keys proving one row per key by `kind:name`; seeds in `one_row_seeds` also get the empty key. + +use crate::compiled_views::_helpers::compiled_models::relation_keys::RELATION_KIND_SEED; +use crate::compiled_views::_helpers::compiled_models::relation_keys::RELATION_KIND_SOURCE; +use crate::compiled_views::_helpers::compiled_models::relation_keys::declared_relation; +use crate::compiled_views::_helpers::compiled_models::relation_keys::source_view_key_row; +use crate::compiled_views::main::compiled_models::entry_relation_key_row::entry_relation_key_row; +use crate::compiled_views::models::RelationKeyRow; +use crate::compiled_views::types::RelationKeys; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_rules::relation_keys::main::declared_relation_keys::declared_relation_keys as derive_keys; +use sqlbuild_rules::relation_keys::models::DeclaredRelation; +use std::collections::BTreeMap; + +/// Keys proving one row per key by `kind:name`; seeds in `one_row_seeds` also get the empty key. +pub fn relation_keys<'py>( + py: Python<'py>, + mut rows: Vec, + (seeds, sources): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + one_row_seeds: &[String], +) -> PyResult { + for (kind, entries) in [(RELATION_KIND_SEED, seeds), (RELATION_KIND_SOURCE, sources)] { + for entry in entries.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + if let Some(row) = source_view_key_row(&entry) { + rows.push(row?); + continue; + } + let name: String = native_field(&entry, "name")?.extract()?; + rows.push(entry_relation_key_row( + kind, + name, + &PyDict::new(py), + &entry, + )?); + } + } + let mut relations: Vec = rows.into_iter().map(declared_relation).collect(); + for relation in &mut relations { + relation.at_most_one_row = + relation.kind == RELATION_KIND_SEED && one_row_seeds.contains(&relation.name); + } + let keys: BTreeMap>> = py.detach(|| derive_keys(&relations)); + Ok(keys.into_iter().collect()) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_location.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_location.rs new file mode 100644 index 0000000000..cd255c841a --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_location.rs @@ -0,0 +1,25 @@ +//! The `CompiledRelationLocation` of a destination. + +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::DATABASE_KEY; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::LOGICAL_DATABASE_ATTRIBUTE; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::LOGICAL_SCHEMA_ATTRIBUTE; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::SCHEMA_KEY; +use crate::compiled_views::models::CompiledViewClasses; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_project::models::DestinationFact; + +/// The `CompiledRelationLocation` of a destination. +pub fn relation_location( + classes: &CompiledViewClasses<'_>, + destination: &DestinationFact, +) -> PyResult> { + let fields: Bound<'_, PyDict> = PyDict::new(classes.relation_location.py()); + fields.set_item(DATABASE_KEY, &destination.database)?; + fields.set_item(SCHEMA_KEY, &destination.schema)?; + fields.set_item("name", &destination.name)?; + fields.set_item("qualified_name", &destination.qualified_name)?; + fields.set_item(LOGICAL_SCHEMA_ATTRIBUTE, &destination.logical_schema)?; + fields.set_item(LOGICAL_DATABASE_ATTRIBUTE, &destination.logical_database)?; + Ok(classes.relation_location.call((), Some(&fields))?.unbind()) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relative_path.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relative_path.rs new file mode 100644 index 0000000000..6354ed8456 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relative_path.rs @@ -0,0 +1,14 @@ +//! The model file's relative path. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The model file's relative path. +pub fn relative_path<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.relative_path(py)?.into_bound(py)); + } + native_field(model, "relative_path") +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/schema_entry.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/schema_entry.rs new file mode 100644 index 0000000000..84c8676b79 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/schema_entry.rs @@ -0,0 +1,14 @@ +//! The model's attached `SchemaModelEntry`, or `None`. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The model's attached `SchemaModelEntry`, or `None`. +pub fn schema_entry<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.schema_entry(py)?.into_bound(py)); + } + native_field(model, "schema_entry") +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/source_key_input.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/source_key_input.rs new file mode 100644 index 0000000000..a9056cd62c --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/source_key_input.rs @@ -0,0 +1,13 @@ +//! What `relation_keys` reads for a source: its native view itself, or its `source_entry`. + +use crate::resource_views::main::source_views::read_source_view::read_source_view; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// What `relation_keys` reads for a source: its native view itself, or its `source_entry`. +pub fn source_key_input<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + if read_source_view(source, |_, _| Ok(())).is_some() { + return Ok(source.clone()); + } + native_field(source, "source_entry") +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/star_flags.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/star_flags.rs new file mode 100644 index 0000000000..d6cb2376aa --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/star_flags.rs @@ -0,0 +1,19 @@ +//! `(has star, star resolved)` of the model's analysed query. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// `(has star, star resolved)` of the model's analysed query. +pub fn star_flags(model: &Bound<'_, PyAny>) -> PyResult<(bool, bool)> { + if let Some(view) = native_model(model)? { + return Ok(( + view.fast_lineage_has_star(model.py())?, + view.fast_lineage_star_resolved(model.py())?, + )); + } + Ok(( + native_field(model, "fast_lineage_has_star")?.is_truthy()?, + native_field(model, "fast_lineage_star_resolved")?.is_truthy()?, + )) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/unchecked_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/unchecked_columns.rs new file mode 100644 index 0000000000..976f0b4572 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/unchecked_columns.rs @@ -0,0 +1,16 @@ +//! Output columns whose types semantic recovery forgot, in any order. + +use crate::compiled_views::main::compiled_models::native_model::native_model; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Output columns whose types semantic recovery forgot, in any order. +pub fn unchecked_columns(model: &Bound<'_, PyAny>) -> PyResult> { + if let Some(view) = native_model(model)? { + return Ok(view.unchecked_columns().to_vec()); + } + native_field(model, "unchecked_output_columns")? + .try_iter()? + .map(|column| column?.extract()) + .collect() +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/mod.rs new file mode 100644 index 0000000000..fe1d52175b --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/mod.rs @@ -0,0 +1,3 @@ +pub mod compiled_models; +pub mod model_inputs; +pub mod project; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_columns.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_columns.rs new file mode 100644 index 0000000000..73b1ed3634 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_columns.rs @@ -0,0 +1,25 @@ +//! The declared column facts of `model_input`'s attached schema, empty without one. + +use crate::compiled_project::main::python_column_facts::python_column_facts; +use crate::compiled_views::models::NativeModelInput; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The declared column facts of `model_input`'s attached schema, empty without one. +pub fn attached_columns(model_input: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = model_input.py(); + if let Ok(native) = model_input.cast::() { + return native.get().read_with_owner(py, |owner, record| { + Ok(owner + .input_schema_fact(record) + .map(|fact| fact.columns.clone()) + .unwrap_or_default()) + }); + } + let entry: Bound<'_, PyAny> = native_field(model_input, "schema_entry")?; + if entry.is_none() { + return Ok(Vec::new()); + } + python_column_facts(&native_field(&entry, "columns")?, None) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_schema.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_schema.rs new file mode 100644 index 0000000000..179767ea94 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/attached_schema.rs @@ -0,0 +1,33 @@ +//! The attached schema of `model_input`; a native input answers without building its entry. + +use crate::compiled_views::models::AttachedSchema; +use crate::compiled_views::models::NativeModelInput; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::model_config::main::headers::family_specs::family_specs; + +/// The attached schema of `model_input`; a native input answers without building its entry. +pub fn attached_schema<'py>(model_input: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model_input.py(); + if let Ok(native) = model_input.cast::() { + return native.get().read(py, |record, py| { + let Some(schema) = &record.schema else { + return Ok(None); + }; + Ok(Some(AttachedSchema { + name: record.name.clone(), + dynamic_columns: schema.host.dynamic_columns.read(py)?, + type_enforcement: schema.type_enforcement, + })) + }); + } + let entry: Bound<'py, PyAny> = native_field(model_input, "schema_entry")?; + if entry.is_none() { + return Ok(None); + } + Ok(Some(AttachedSchema { + name: native_field(&entry, "name")?.extract()?, + dynamic_columns: family_specs(&native_field(&entry, "dynamic_columns")?)?, + type_enforcement: native_field(&entry, "type_enforcement")?.is_truthy()?, + })) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_analysis_sql.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_analysis_sql.rs new file mode 100644 index 0000000000..8b1ecd7ee1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_analysis_sql.rs @@ -0,0 +1,16 @@ +//! The analysis text of `model_input`: its compiled query with each typed SQL macro call marked. + +use crate::compiled_views::models::NativeModelInput; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::sql_macros::main::typed_analysis_sql::typed_analysis_sql; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The analysis text of `model_input`: its compiled query with each typed SQL macro call marked. +pub fn input_analysis_sql(model_input: &Bound<'_, PyAny>, dialect: &str) -> PyResult { + if let Ok(native) = model_input.cast::() { + return native.get().read(model_input.py(), |record, _| { + Ok(typed_analysis_sql(&record.query_sql, &record.typed_calls, dialect).0) + }); + } + native_field(model_input, "query_sql")?.extract() +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_model_name.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_model_name.rs new file mode 100644 index 0000000000..8d0adf5eb8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_model_name.rs @@ -0,0 +1,18 @@ +//! The name of `model_input`, its file stem; a native input answers from its record. + +use crate::compiled_views::models::NativeModelInput; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The name of `model_input`, its file stem; a native input answers from its record. +pub fn input_model_name(model_input: &Bound<'_, PyAny>) -> PyResult { + if let Ok(native) = model_input.cast::() { + return native + .get() + .read(model_input.py(), |record, _| Ok(record.name.clone())); + } + native_field(model_input, "model_file")? + .getattr("file_path")? + .getattr("stem")? + .extract() +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_reference_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_reference_rows.rs new file mode 100644 index 0000000000..9d51115d52 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_reference_rows.rs @@ -0,0 +1,21 @@ +//! Each reference of `model_input` as `(kind, name, package, call argument count)`. + +use crate::compiled_views::_helpers::model_inputs::model_input_view::reference_row_of; +use crate::compiled_views::models::NativeModelInput; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_project::models::ReferenceRow; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Each reference of `model_input` as `(kind, name, package, call argument count)`. +pub fn input_reference_rows(model_input: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = model_input.py(); + if let Ok(native) = model_input.cast::() { + return native + .get() + .read(py, |record, _| Ok(record.references.clone())); + } + native_field(model_input, "references")? + .try_iter()? + .map(|reference| reference_row_of(&reference?)) + .collect() +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_sql_macro_checks.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_sql_macro_checks.rs new file mode 100644 index 0000000000..adbf94a3dd --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_sql_macro_checks.rs @@ -0,0 +1,15 @@ +//! What the analysed SQL of a native model input must show for its SQL macro calls to fit. + +use crate::compiled_views::models::NativeModelInput; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; + +/// What the analysed SQL of a native model input must show for its SQL macro calls to fit. +pub fn input_sql_macro_checks(model_input: &Bound<'_, PyAny>) -> PyResult> { + if let Ok(native) = model_input.cast::() { + return native.get().read(model_input.py(), |record, _| { + Ok(record.sql_macro_checks.clone()) + }); + } + Ok(Vec::new()) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/mod.rs new file mode 100644 index 0000000000..e6822ef688 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/mod.rs @@ -0,0 +1,8 @@ +pub mod attached_columns; +pub mod attached_schema; +pub mod input_analysis_sql; +pub mod input_model_name; +pub mod input_reference_rows; +pub mod input_sql_macro_checks; +pub mod reference_tuple; +pub mod view_classes; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs new file mode 100644 index 0000000000..6c1f3cd3ef --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs @@ -0,0 +1,36 @@ +//! `CompileSqlReference` records of `rows`, built once into `cache`. + +use crate::compiled_views::models::InputViewClasses; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_project::models::ReferenceRow; + +/// `CompileSqlReference` records of `rows`, built once into `cache`. +pub fn reference_tuple( + classes: &InputViewClasses<'_>, + rows: &[ReferenceRow], + cache: &PyOnceLock>, +) -> PyResult> { + let py: Python<'_> = classes.sql_reference.py(); + cache + .get_or_try_init(py, || { + let mut references: Vec> = Vec::with_capacity(rows.len()); + for ReferenceRow { + kind, + name, + package, + call_argument_count: count, + } in rows + { + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("ref_kind", classes.reference_kind.call1((kind,))?)?; + fields.set_item("ref_name", name)?; + fields.set_item("ref_package", package)?; + fields.set_item("call_argument_count", count)?; + references.push(classes.sql_reference.call((), Some(&fields))?); + } + Ok::<_, pyo3::PyErr>(PyTuple::new(py, references)?.unbind()) + }) + .map(|references| references.clone_ref(py)) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/view_classes.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/view_classes.rs new file mode 100644 index 0000000000..092dde72df --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/view_classes.rs @@ -0,0 +1,13 @@ +//! The classes the owner's views project attributes with. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::InputViewClasses; +use pyo3::prelude::{Py, PyAnyMethods, PyResult, Python}; + +/// The classes the owner's views project attributes with. +pub fn view_classes<'py>( + owner: &Py, + py: Python<'py>, +) -> PyResult> { + owner.bind(py).borrow().input_classes(py)?.extract() +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/audits.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/audits.rs new file mode 100644 index 0000000000..a7956469d9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/audits.rs @@ -0,0 +1,15 @@ +//! The project's compiled audits. + +use crate::compiled_views::_helpers::project::project_reads::AUDITS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::native_project; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The project's compiled audits. +pub fn audits<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.audits(py).into_bound(py)); + } + native_field(project, AUDITS_FIELD) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/collection.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/collection.rs new file mode 100644 index 0000000000..a4f8bca998 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/collection.rs @@ -0,0 +1,26 @@ +//! The project's resource collection named `field`; any other field is read by attribute. + +use crate::compiled_views::_helpers::project::project_reads::AUDITS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::FUNCTIONS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::MODELS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::SEEDS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::SOURCES_FIELD; +use crate::compiled_views::main::project::audits::audits; +use crate::compiled_views::main::project::compiled_models::compiled_models; +use crate::compiled_views::main::project::functions::functions; +use crate::compiled_views::main::project::seeds::seeds; +use crate::compiled_views::main::project::sources::sources; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The project's resource collection named `field`; any other field is read by attribute. +pub fn collection<'py>(project: &Bound<'py, PyAny>, field: &str) -> PyResult> { + match field { + MODELS_FIELD => compiled_models(project), + SOURCES_FIELD => sources(project), + SEEDS_FIELD => seeds(project), + FUNCTIONS_FIELD => functions(project), + AUDITS_FIELD => audits(project), + _ => native_field(project, field), + } +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compile_inputs_view.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compile_inputs_view.rs new file mode 100644 index 0000000000..6c092c4f49 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compile_inputs_view.rs @@ -0,0 +1,37 @@ +//! The native owner's inputs view when it holds every model input; else a record. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::NativeInputsView; +use crate::compiled_views::models::NativeModelInput; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{IntoPyObjectExt, pyfunction}; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; + +/// The native owner's inputs view when it holds every model input; else a record. +#[pyfunction] +#[pyo3(signature = (*, fields, native_project))] +pub fn compile_inputs_view<'py>( + fields: &Bound<'py, PyDict>, + native_project: Option>, +) -> PyResult> { + let py: Python<'py> = fields.py(); + if let Some(project) = &native_project { + let mut native: bool = true; + for model in fields.as_any().get_item("model_inputs")?.try_iter()? { + if !model?.is_instance_of::() { + native = false; + break; + } + } + if native { + let view: NativeInputsView = + NativeCompiledProject::inputs_view(project.clone(), fields.clone())?; + return view.into_bound_py_any(py); + } + } + let keywords: Bound<'py, PyDict> = fields.copy()?; + keywords.set_item("native_project", native_project)?; + new_record(py, RecordId::CompileProjectInputs, (), Some(&keywords)) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_audit_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_audit_record.rs new file mode 100644 index 0000000000..7d86acd4e1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_audit_record.rs @@ -0,0 +1,12 @@ +//! The audit as a `CompiledAudit`, projected from its owner when it is a native view. + +use crate::compiled_views::_helpers::project::records::record; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::RecordId; + +/// The audit as a `CompiledAudit`, projected from its owner when it is a native view. +#[pyfunction] +pub fn compiled_audit_record<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + record(audit, RecordId::CompiledAudit) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_function_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_function_record.rs new file mode 100644 index 0000000000..c7aca51088 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_function_record.rs @@ -0,0 +1,12 @@ +//! The function as a `CompiledFunction`, projected from its owner when it is a native view. + +use crate::compiled_views::_helpers::project::records::record; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::RecordId; + +/// The function as a `CompiledFunction`, projected from its owner when it is a native view. +#[pyfunction] +pub fn compiled_function_record<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + record(function, RecordId::CompiledFunction) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_model_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_model_record.rs new file mode 100644 index 0000000000..bacd080555 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_model_record.rs @@ -0,0 +1,12 @@ +//! The model as a `CompiledModel`, projected from its owner when it is a native view. + +use crate::compiled_views::_helpers::project::records::record; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::RecordId; + +/// The model as a `CompiledModel`, projected from its owner when it is a native view. +#[pyfunction] +pub fn compiled_model_record<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + record(model, RecordId::CompiledModel) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_models.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_models.rs new file mode 100644 index 0000000000..7052ab01eb --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_models.rs @@ -0,0 +1,15 @@ +//! The project's compiled models, in compile order. + +use crate::compiled_views::_helpers::project::project_reads::MODELS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::native_project; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The project's compiled models, in compile order. +pub fn compiled_models<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.models(py).into_bound(py).into_any()); + } + native_field(project, MODELS_FIELD) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_project_facts.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_project_facts.rs new file mode 100644 index 0000000000..c6f60aa64c --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_project_facts.rs @@ -0,0 +1,23 @@ +//! The compile's retained project, or one recorded from a hand-built project. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::_helpers::project::records::hand_built_project; +use crate::compiled_views::models::NativeProjectView; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compile's retained project, or one recorded from a hand-built project. +#[pyfunction] +pub fn compiled_project_facts<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult> { + if let Ok(view) = project.downcast::() { + return Ok(view.get().owner().bind(project.py()).clone()); + } + let retained: Bound<'py, PyAny> = native_field(project, "native_project")?; + if !retained.is_none() { + return Ok(retained.downcast_into()?); + } + hand_built_project(project) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_seed_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_seed_record.rs new file mode 100644 index 0000000000..0bb27207d6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_seed_record.rs @@ -0,0 +1,12 @@ +//! The seed as a `CompiledSeed`, projected from its owner when it is a native view. + +use crate::compiled_views::_helpers::project::records::record; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::RecordId; + +/// The seed as a `CompiledSeed`, projected from its owner when it is a native view. +#[pyfunction] +pub fn compiled_seed_record<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + record(seed, RecordId::CompiledSeed) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_source_record.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_source_record.rs new file mode 100644 index 0000000000..e7c8678f22 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_source_record.rs @@ -0,0 +1,12 @@ +//! The source as a `CompiledSource`, projected from its owner when it is a native view. + +use crate::compiled_views::_helpers::project::records::record; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::types::RecordId; + +/// The source as a `CompiledSource`, projected from its owner when it is a native view. +#[pyfunction] +pub fn compiled_source_record<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + record(source, RecordId::CompiledSource) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows.rs new file mode 100644 index 0000000000..91431fa49b --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows.rs @@ -0,0 +1,26 @@ +//! Enum and constant declarations as native rows, in the given order. + +use crate::compiled_views::_helpers::project::records::constant_row; +use crate::compiled_views::_helpers::project::records::enum_row; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyList, PyTuple}; + +/// Enum and constant declarations as native rows, in the given order. +#[pyfunction] +#[pyo3(signature = (*, enums, constants))] +pub fn declaration_rows<'py>( + enums: &Bound<'py, PyAny>, + constants: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyList>, Bound<'py, PyList>)> { + let py: Python<'py> = enums.py(); + let enum_rows: Vec> = enums + .try_iter()? + .map(|declaration| enum_row(&declaration?)) + .collect::>()?; + let constant_rows: Vec> = constants + .try_iter()? + .map(|declaration| constant_row(&declaration?)) + .collect::>()?; + Ok((PyList::new(py, enum_rows)?, PyList::new(py, constant_rows)?)) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows_function.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows_function.rs new file mode 100644 index 0000000000..2752047bb0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/declaration_rows_function.rs @@ -0,0 +1,10 @@ +//! `declaration_rows` as a Python function. + +use crate::compiled_views::main::project::declaration_rows::declaration_rows; + +/// `declaration_rows` as a Python function. +pub fn declaration_rows_function( + py: pyo3::Python<'_>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(declaration_rows, py) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/functions.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/functions.rs new file mode 100644 index 0000000000..8726bb548f --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/functions.rs @@ -0,0 +1,15 @@ +//! The project's compiled functions. + +use crate::compiled_views::_helpers::project::project_reads::FUNCTIONS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::native_project; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The project's compiled functions. +pub fn functions<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.functions(py).into_bound(py)); + } + native_field(project, FUNCTIONS_FIELD) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/inputs_with.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/inputs_with.rs new file mode 100644 index 0000000000..ea9d9baacf --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/inputs_with.rs @@ -0,0 +1,17 @@ +//! The compile inputs with `changes`; a native view shares its owner's model inputs. + +use crate::compiled_views::_helpers::project::records::with_changes; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::records::types::RecordId; + +/// The compile inputs with `changes`; a native view shares its owner's model inputs. +#[pyfunction] +#[pyo3(signature = (*, inputs, **changes))] +pub fn inputs_with<'py>( + inputs: &Bound<'py, PyAny>, + changes: Option>, +) -> PyResult> { + with_changes(inputs, RecordId::CompileProjectInputs, changes) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/mod.rs new file mode 100644 index 0000000000..f18ad74f48 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/mod.rs @@ -0,0 +1,24 @@ +pub mod audits; +pub mod collection; +pub mod compile_inputs_view; +pub mod compiled_audit_record; +pub mod compiled_function_record; +pub mod compiled_model_record; +pub mod compiled_models; +pub mod compiled_project_facts; +pub mod compiled_seed_record; +pub mod compiled_source_record; +pub mod declaration_rows; +pub mod declaration_rows_function; +pub mod functions; +pub mod inputs_with; +pub mod plain_typed_value; +pub mod project_with; +pub mod record_model_analyses; +pub mod register_records; +pub mod replace; +pub mod seeds; +pub mod sources; +pub mod sql_tests; +pub mod with_binding_diagnostics; +pub mod with_type_recovery; diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs new file mode 100644 index 0000000000..653df3def8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs @@ -0,0 +1,36 @@ +//! A typed SQL value as the plain value native consumers read. + +use crate::compiled_views::_helpers::project::records::COLLECTION_KINDS; +use crate::compiled_views::_helpers::project::records::DECIMAL_KIND; +use crate::compiled_views::_helpers::project::records::SCALAR_KINDS; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyList}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A typed SQL value as the plain value native consumers read. +#[pyfunction] +pub fn plain_typed_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let kind: String = native_field(value, "kind")?.getattr("value")?.extract()?; + let inner: Bound<'py, PyAny> = native_field(value, "value")?; + if kind == DECIMAL_KIND { + return Ok(inner.str()?.into_any()); + } + if SCALAR_KINDS.contains(&kind.as_str()) { + return Ok(inner); + } + if COLLECTION_KINDS.contains(&kind.as_str()) { + let items: Vec> = inner + .try_iter()? + .map(|item| plain_typed_value(&item?)) + .collect::>()?; + return Ok(PyList::new(py, items)?.into_any()); + } + let mapping: Bound<'py, PyDict> = PyDict::new(py); + for pair in inner.try_iter()? { + let (key, item): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + mapping.set_item(key, plain_typed_value(&item)?)?; + } + Ok(mapping.into_any()) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/project_with.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/project_with.rs new file mode 100644 index 0000000000..2265e0dcf0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/project_with.rs @@ -0,0 +1,17 @@ +//! The project with `changes`; a native view shares its owner's models. + +use crate::compiled_views::_helpers::project::records::with_changes; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::records::types::RecordId; + +/// The project with `changes`; a native view shares its owner's models. +#[pyfunction] +#[pyo3(signature = (*, project, **changes))] +pub fn project_with<'py>( + project: &Bound<'py, PyAny>, + changes: Option>, +) -> PyResult> { + with_changes(project, RecordId::CompiledProject, changes) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/record_model_analyses.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/record_model_analyses.rs new file mode 100644 index 0000000000..bddd236ce2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/record_model_analyses.rs @@ -0,0 +1,75 @@ +//! Transfer each assembled model's typed analysis to its retained facts; return how many. + +use crate::compiled_project::models::{AnalysisPy, NativeCompiledProject, ProofPy}; +use crate::compiled_views::_helpers::project::records::family_types; +use crate::compiled_views::models::NativeCompiledModel; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +/// Transfer each assembled model's typed analysis to its retained facts; return how many. +#[pyfunction] +#[pyo3(signature = (*, project, models))] +pub fn record_model_analyses<'py>( + project: &Bound<'py, NativeCompiledProject>, + models: &Bound<'py, PyAny>, +) -> PyResult { + let py: Python<'py> = project.py(); + let models: Vec> = models.try_iter()?.collect::>()?; + let views: Vec> = models + .iter() + .filter(|model| model.is_instance_of::()) + .map(|model| model.extract::>()) + .collect::>()?; + let mut transferred: usize = NativeCompiledProject::record_view_analyses(project, views)?; + let nullable: Bound<'py, PyAny> = enum_member(py, RecordId::InferredNullability, "NULLABLE")?; + for model in &models { + if model.is_instance_of::() { + continue; + } + let inferred: Bound<'py, PyAny> = native_field(model, "inferred_rows")?; + let rows: Vec<(String, Option, Bound<'py, PyAny>)> = if inferred.is_none() { + Vec::new() + } else { + inferred.extract()? + }; + let proof: Bound<'py, PyAny> = native_field(model, "dynamic_column_contract")?; + let proof = if proof.is_none() { + None + } else { + let families: Bound<'py, PyAny> = native_field(&proof, "families")?; + Some(ProofPy { + output_proven: native_field(&proof, "output_proven")?.extract()?, + bare_dynamic_pivot: native_field(&proof, "bare_dynamic_pivot")?.extract()?, + families: family_types(&families, true)?, + failure_reason: native_field(&proof, "failure_reason")?.extract()?, + named_families: family_types(&families, false)?, + }) + }; + let analysis = AnalysisPy { + inferred: !inferred.is_none(), + inferred_columns: rows + .iter() + .map(|(name, column_type, _)| (name.clone(), column_type.clone())) + .collect(), + nullable_columns: rows + .iter() + .map(|(_, _, nullability)| nullability.eq(&nullable)) + .collect::>()?, + fast_lineage_has_star: native_field(model, "fast_lineage_has_star")?.is_truthy()?, + unchecked: native_field(model, "unchecked_output_columns")? + .try_iter()? + .map(|column| column?.extract()) + .collect::>()?, + }; + let name: String = native_field(model, "name")?.extract()?; + transferred += usize::from( + project + .borrow_mut() + .record_model_analysis(&name, analysis, proof), + ); + } + Ok(transferred) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/register_records.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/register_records.rs new file mode 100644 index 0000000000..9a0aa56f4f --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/register_records.rs @@ -0,0 +1,43 @@ +//! Register the records functions and classes. + +use crate::compiled_views::_helpers::project::records::compiled_project_record; +use crate::compiled_views::main::project::compile_inputs_view::compile_inputs_view; +use crate::compiled_views::main::project::compiled_audit_record::compiled_audit_record; +use crate::compiled_views::main::project::compiled_function_record::compiled_function_record; +use crate::compiled_views::main::project::compiled_model_record::compiled_model_record; +use crate::compiled_views::main::project::compiled_project_facts::compiled_project_facts; +use crate::compiled_views::main::project::compiled_seed_record::compiled_seed_record; +use crate::compiled_views::main::project::compiled_source_record::compiled_source_record; +use crate::compiled_views::main::project::declaration_rows::declaration_rows; +use crate::compiled_views::main::project::inputs_with::inputs_with; +use crate::compiled_views::main::project::plain_typed_value::plain_typed_value; +use crate::compiled_views::main::project::project_with::project_with; +use crate::compiled_views::main::project::record_model_analyses::record_model_analyses; +use crate::compiled_views::main::project::with_binding_diagnostics::with_binding_diagnostics; +use crate::compiled_views::main::project::with_type_recovery::with_type_recovery; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the records functions and classes. +pub fn register_records(module: &Bound<'_, PyModule>) -> PyResult<()> { + for function in [ + wrap_pyfunction!(compiled_model_record, module)?, + wrap_pyfunction!(compiled_audit_record, module)?, + wrap_pyfunction!(compiled_function_record, module)?, + wrap_pyfunction!(compiled_source_record, module)?, + wrap_pyfunction!(compiled_seed_record, module)?, + wrap_pyfunction!(compiled_project_record, module)?, + wrap_pyfunction!(with_binding_diagnostics, module)?, + wrap_pyfunction!(with_type_recovery, module)?, + wrap_pyfunction!(project_with, module)?, + wrap_pyfunction!(inputs_with, module)?, + wrap_pyfunction!(compile_inputs_view, module)?, + wrap_pyfunction!(plain_typed_value, module)?, + wrap_pyfunction!(declaration_rows, module)?, + wrap_pyfunction!(record_model_analyses, module)?, + wrap_pyfunction!(compiled_project_facts, module)?, + ] { + module.add_function(function)?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs new file mode 100644 index 0000000000..fe804a6c28 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs @@ -0,0 +1,19 @@ +//! `dataclasses.replace(value, **changes)`. + +use crate::compiled_views::_helpers::project::records::DATACLASSES_MODULE; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::boundary::main::imported::imported; + +/// `dataclasses.replace(value, **changes)`. +pub fn replace<'py>( + value: &Bound<'py, PyAny>, + changes: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let py: Python<'py> = value.py(); + let keywords: Bound<'py, PyDict> = PyDict::new(py); + for (name, change) in changes { + keywords.set_item(*name, change)?; + } + imported(py, DATACLASSES_MODULE, "replace")?.call((value,), Some(&keywords)) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/seeds.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/seeds.rs new file mode 100644 index 0000000000..41c2f79b61 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/seeds.rs @@ -0,0 +1,15 @@ +//! The project's compiled seeds. + +use crate::compiled_views::_helpers::project::project_reads::SEEDS_FIELD; +use crate::compiled_views::_helpers::project::project_reads::native_project; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The project's compiled seeds. +pub fn seeds<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.seeds(py).into_bound(py)); + } + native_field(project, SEEDS_FIELD) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/sources.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/sources.rs new file mode 100644 index 0000000000..b9264ac789 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/sources.rs @@ -0,0 +1,15 @@ +//! The project's compiled sources. + +use crate::compiled_views::_helpers::project::project_reads::SOURCES_FIELD; +use crate::compiled_views::_helpers::project::project_reads::native_project; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The project's compiled sources. +pub fn sources<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.sources(py).into_bound(py)); + } + native_field(project, SOURCES_FIELD) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/sql_tests.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/sql_tests.rs new file mode 100644 index 0000000000..5a7f8b71c4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/sql_tests.rs @@ -0,0 +1,14 @@ +//! The project's compiled SQL tests. + +use crate::compiled_views::_helpers::project::project_reads::native_project; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The project's compiled SQL tests. +pub fn sql_tests<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.sql_tests(py).into_bound(py)); + } + native_field(project, "sql_tests") +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/with_binding_diagnostics.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/with_binding_diagnostics.rs new file mode 100644 index 0000000000..64e9ccc734 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/with_binding_diagnostics.rs @@ -0,0 +1,21 @@ +//! The model with these final binding diagnostics. + +use crate::compiled_views::main::project::replace::replace; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::types::RecordId; + +/// The model with these final binding diagnostics. +#[pyfunction] +#[pyo3(signature = (*, model, diagnostics))] +pub fn with_binding_diagnostics<'py>( + model: &Bound<'py, PyAny>, + diagnostics: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = model.py(); + if model.is_instance(&record_class(py, RecordId::CompiledModel)?)? { + return replace(model, &[("binding_diagnostics", diagnostics.clone())]); + } + model.call_method1("with_binding_diagnostics", (diagnostics,)) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/project/with_type_recovery.rs b/crates/sqlbuild-python-project/src/compiled_views/main/project/with_type_recovery.rs new file mode 100644 index 0000000000..8b6842ce90 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/main/project/with_type_recovery.rs @@ -0,0 +1,58 @@ +//! The model with its recovered bindings and its poisoned outputs' types forgotten. + +use crate::compiled_views::main::project::replace::replace; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyFrozenSet, PyFrozenSetMethods, PyTuple}; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +/// The model with its recovered bindings and its poisoned outputs' types forgotten. +#[pyfunction] +#[pyo3(signature = (*, model, diagnostics, poisoned))] +pub fn with_type_recovery<'py>( + model: &Bound<'py, PyAny>, + diagnostics: &Bound<'py, PyAny>, + poisoned: &Bound<'py, PyFrozenSet>, +) -> PyResult> { + let py: Python<'py> = model.py(); + let name: String = native_field(model, "name")?.extract()?; + let mut unchecked: Vec = Vec::new(); + for pair in poisoned.iter() { + let (model_name, column): (String, String) = pair.extract()?; + if model_name == name { + unchecked.push(column); + } + } + if !model.is_instance(&record_class(py, RecordId::CompiledModel)?)? { + return model.call_method1("with_type_recovery", (diagnostics, unchecked)); + } + let inferred: Bound<'py, PyAny> = native_field(model, "inferred_columns")?; + let columns: Bound<'py, PyAny> = if inferred.is_none() { + inferred + } else { + let mut columns: Vec> = Vec::new(); + for column in inferred.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let key = (name.as_str(), native_field(&column, "name")?); + columns.push(if poisoned.contains(key)? { + replace(&column, &[("type", py.None().into_bound(py))])? + } else { + column + }); + } + PyTuple::new(py, columns)?.into_any() + }; + replace( + model, + &[ + ("binding_diagnostics", diagnostics.clone()), + ("inferred_columns", columns), + ( + "unchecked_output_columns", + PyFrozenSet::new(py, &unchecked)?.into_any(), + ), + ], + ) +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/mod.rs b/crates/sqlbuild-python-project/src/compiled_views/mod.rs new file mode 100644 index 0000000000..937dcc3deb --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/mod.rs @@ -0,0 +1,7 @@ +//! Python views of the compiled project: its models, model inputs, analysis, records and relation keys. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/compiled_views/models.rs b/crates/sqlbuild-python-project/src/compiled_views/models.rs new file mode 100644 index 0000000000..28438e56d5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/models.rs @@ -0,0 +1,1696 @@ +//! Public records of compiled views. + +use crate::compiled_project::main::reference_row_tuple::reference_row_tuple; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_project::models::ReferenceTuple; +use crate::compiled_views::_helpers::compiled_models::compiled_model_analysis::AnalysisRecord; +use crate::compiled_views::_helpers::compiled_models::compiled_model_analysis::AnalysisViews; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::AuditedSchema; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::CompiledArguments; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::CompiledRecord; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::DestinationRow; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::MODEL_RESOURCE; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::ModelOverlay; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::NULLABLE; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::SQL_ANALYSIS_KEY; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::ViewProjections; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::casefolded_families; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::compiled_of; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::compiled_record; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::lineage_input; +use crate::compiled_views::_helpers::compiled_models::compiled_model_view::model_name; +use crate::compiled_views::_helpers::model_inputs::inputs_view::InputsFields; +use crate::compiled_views::_helpers::model_inputs::model_input_view::attached_schema_name; +use crate::compiled_views::_helpers::model_inputs::model_input_view::authored_expansion; +use crate::compiled_views::_helpers::model_inputs::model_input_view::expansion_object; +use crate::compiled_views::_helpers::model_inputs::model_input_view::model_relation_key_row; +use crate::compiled_views::_helpers::model_inputs::model_input_view::query_sql_text; +use crate::compiled_views::_helpers::model_inputs::model_input_view::read_input; +use crate::compiled_views::_helpers::model_inputs::model_input_view::read_input_with_owner; +use crate::compiled_views::_helpers::model_inputs::model_input_view::references; +use crate::compiled_views::_helpers::model_inputs::model_input_view::rejected_opt_out; +use crate::compiled_views::_helpers::model_inputs::model_input_view::schema_entry; +use crate::compiled_views::_helpers::project::project_view::CompilerHandles; +use crate::compiled_views::_helpers::project::project_view::ProjectFields; +use crate::compiled_views::_helpers::project::project_view::SCHEMA_ENTRY_ATTRIBUTE; +use crate::compiled_views::_helpers::project::project_view::TAGS_ATTRIBUTE; +use crate::compiled_views::_helpers::project::project_view::resource_row; +use crate::compiled_views::main::compiled_models::defaulted_destination::defaulted_destination; +use crate::compiled_views::main::compiled_models::model_graph_row::model_graph_row; +use crate::compiled_views::main::compiled_models::object_key::object_key; +use crate::compiled_views::main::compiled_models::relation_location::relation_location; +use crate::compiled_views::main::model_inputs::view_classes::view_classes; +use crate::compiled_views::types::DeclaredRow; +use crate::compiled_views::types::LineageInput; +use crate::compiled_views::types::ModelInputRecord; +use crate::compiled_views::types::RelationKeyTuple; +use crate::graph::main::project::project_graph::project_graph; +use crate::graph::models::NativeProjectGraph; +use crate::graph::models::ResourceRow; +use crate::resource_views::models::AuditHandles; +use crate::resource_views::models::AuditProjections; +use crate::resource_views::models::FunctionHandles; +use crate::resource_views::models::FunctionProjections; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; +use crate::resource_views::models::SeedHandles; +use crate::resource_views::models::SourceHandles; +use crate::resource_views::models::SourceProjections; +use pyo3::FromPyObject; +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyFrozenSet; +use pyo3::types::PyString; +use pyo3::types::PyTuple; +use pyo3::types::PyTupleMethods; +use sqlbuild_analysis::assembly::analysis_session::main::contract_proof_json::contract_proof_json; +use sqlbuild_analysis::assembly::analysis_session::models::LineageRow; +use sqlbuild_analysis::compiled_project::main::model_facts::model_facts; +use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; +use sqlbuild_analysis::compiled_project::models::DynamicProofFact; +use sqlbuild_analysis::compiled_project::models::ModelAnalysisFact; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; +use sqlbuild_project::models::DestinationFact; +use sqlbuild_project::models::ExpansionRecord; +use sqlbuild_project::models::ModelMetadataTexts; +use sqlbuild_project::types::InferredRow; +use sqlbuild_project::types::ProjectHost; +use sqlbuild_python_base::compile_driver::constants::LINEAGE_GRAPH_BUILDS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::model_config::models::HeaderAudits; +use sqlbuild_python_front::model_config::models::HeaderColumns; +use sqlbuild_python_front::model_config::models::HeaderFamilies; +use std::collections::BTreeMap; +use std::fmt::Debug; +use std::path::PathBuf; +use std::sync::Arc; + +/// Python projections of the analysis, each built once on first access. +pub struct AnalysisProjections { + pub(crate) lineage: PyOnceLock>, + pub(crate) binding: PyOnceLock>, + pub(crate) proof: PyOnceLock>, +} + +impl Default for AnalysisProjections { + fn default() -> Self { + Self { + lineage: PyOnceLock::new(), + binding: PyOnceLock::new(), + proof: PyOnceLock::new(), + } + } +} + +/// A model's SQL rule inputs; `excluded` is `None` when its config values could not be read natively. +#[derive(Debug)] +pub struct SqlRuleInputs { + pub posix_path: String, + pub excluded: Option, + pub key: Option, +} + +/// What a model's SQL rule cache key reads. +#[derive(Debug)] +pub struct SqlRuleKey { + pub authored_sql: String, + pub query_sql: String, + /// The canonical JSON text of the model's dynamic pivot proof. + pub contract: Option, +} + +/// The Python classes a compiled model's projected attributes are built with. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub struct CompiledViewClasses<'py> { + pub(crate) object_key: Bound<'py, PyAny>, + pub(crate) relation_location: Bound<'py, PyAny>, + pub inferred_column: Bound<'py, PyAny>, + pub nullability: Bound<'py, PyAny>, + pub compiler_diagnostic: Bound<'py, PyAny>, + pub source_location: Bound<'py, PyAny>, + pub compact_lineage_facts: Bound<'py, PyAny>, + pub lineage_column_fact: Bound<'py, PyAny>, + pub lineage_source_fact: Bound<'py, PyAny>, + pub contract_proof: Bound<'py, PyAny>, + pub family_proof: Bound<'py, PyAny>, +} + +/// A compiled model's Python attribute projections, each built once on first access. +pub struct CompiledProjections { + pub(crate) key: PyOnceLock>, + pub(crate) deps: PyOnceLock>, +} + +impl Default for CompiledProjections { + fn default() -> Self { + Self { + key: PyOnceLock::new(), + deps: PyOnceLock::new(), + } + } +} + +/// One compiled model, projected from the owner on demand. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeCompiledModel { + pub(crate) owner: Py, + pub(crate) index: usize, + pub(crate) overlay: ModelOverlay, + pub(crate) projections: ViewProjections, +} + +impl NativeCompiledModel { + pub fn new(owner: Py, index: usize) -> Self { + Self::with_overlay(owner, index, ModelOverlay::default()) + } + + pub(crate) fn with_overlay( + owner: Py, + index: usize, + overlay: ModelOverlay, + ) -> Self { + Self { + owner, + index, + overlay, + projections: ViewProjections::default(), + } + } + + pub(crate) fn changed(&self, py: Python<'_>, change: impl FnOnce(&mut ModelOverlay)) -> Self { + let mut overlay: ModelOverlay = self.overlay.clone_ref(py); + change(&mut overlay); + Self::with_overlay(self.owner.clone_ref(py), self.index, overlay) + } + + pub fn read<'py, T>( + &self, + py: Python<'py>, + read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, + ) -> PyResult { + read_input(&self.owner, self.index, py, read) + } + + pub(crate) fn compiled<'py, T>( + &self, + py: Python<'py>, + read: impl FnOnce(&CompiledRecord, Python<'py>) -> PyResult, + ) -> PyResult { + self.read(py, |record, py| read(compiled_of(record)?, py)) + } + + /// Read the effective destination, including namespace overlays, without a Python projection. + pub fn read_destination( + &self, + py: Python<'_>, + read: impl FnOnce(&DestinationFact) -> PyResult, + ) -> PyResult { + self.compiled(py, |compiled, _| { + read( + self.overlay + .destination + .as_ref() + .unwrap_or(&compiled.destination), + ) + }) + } + + /// This view's typed analysis as the owner's model facts read it, and the model's name. + pub fn analysis_fact(&self, py: Python<'_>) -> PyResult<(String, ModelAnalysisFact)> { + self.read(py, |record, _| { + let analysis: &AnalysisRecord = &compiled_of(record)?.analysis; + let rows: &[InferredRow] = analysis.inferred.as_deref().unwrap_or_default(); + let mut inferred_columns: Vec<(String, Option)> = + Vec::with_capacity(rows.len()); + for (name, column_type, _) in rows { + let poisoned: bool = self.overlay.poisoned.contains(name); + inferred_columns.push((name.clone(), column_type.clone().filter(|_| !poisoned))); + } + let dynamic_proof: Option = + analysis.proof.as_ref().map(|proof| DynamicProofFact { + output_proven: proof.output_proven, + bare_dynamic_pivot: proof.bare_dynamic_pivot, + families: casefolded_families(&proof.families), + failure_reason: proof.failure_reason.clone(), + named_families: proof.families.clone(), + }); + let fact: ModelAnalysisFact = ModelAnalysisFact { + inferred_columns, + dynamic_proof, + inferred: analysis.inferred.is_some(), + nullable_columns: rows + .iter() + .map(|(_, _, nullability)| nullability == NULLABLE) + .collect(), + fast_lineage_has_star: analysis.has_star, + unchecked_output_columns: self.overlay.poisoned.clone(), + }; + Ok((model_name(record), fact)) + }) + } + + /// Each output column of the kept lineage with its `(resource name, column)` sources. + pub fn lineage_inputs(&self, py: Python<'_>) -> PyResult>> { + self.compiled(py, |compiled, _| { + Ok(compiled + .analysis + .lineage_rows() + .map(|rows| rows.iter().map(lineage_input).collect())) + }) + } + + /// Whether the model's dynamic pivot proof proved its output columns. + pub fn output_proven(&self, py: Python<'_>) -> PyResult { + self.compiled(py, |compiled, _| { + Ok(compiled + .analysis + .proof + .as_ref() + .is_some_and(|proof| proof.output_proven)) + }) + } + + /// Output columns whose types semantic recovery forgot. + pub fn unchecked_columns(&self) -> &[String] { + &self.overlay.poisoned + } + + /// The kept lineage rows with their transform and confidence codes, or `None`. + pub fn lineage_rows(&self, py: Python<'_>) -> PyResult>> { + self.compiled(py, |compiled, _| { + Ok(compiled.analysis.lineage_rows().map(<[LineageRow]>::to_vec)) + }) + } + + /// The input relations of a dynamic pivot proof that proved its outputs, else `None`. + pub fn proven_inputs(&self, py: Python<'_>) -> PyResult>> { + self.compiled(py, |compiled, _| { + Ok(compiled + .analysis + .proof + .as_ref() + .filter(|proof| proof.output_proven) + .map(|proof| proof.input_relations.clone())) + }) + } + + /// The attached schema's declared `SchemaColumn`s, or `None` without a schema. + pub fn declared_columns(&self, py: Python<'_>) -> PyResult>> { + self.read(py, |record, py| { + record + .schema + .as_ref() + .map(|schema| schema.host.columns.project(py).map(Bound::unbind)) + .transpose() + }) + } + + /// The attached schema's entry audits, its column specs and each audited column's index and name. + pub fn schema_audits_and_columns(&self, py: Python<'_>) -> PyResult> { + read_input_with_owner(&self.owner, self.index, py, |owner, record| { + let (Some(schema), Some(fact)) = (&record.schema, owner.input_schema_fact(record)) + else { + return Ok(None); + }; + let audited: Vec<(usize, String)> = fact + .columns + .iter() + .enumerate() + .filter(|(_, column)| !column.audit_definitions.is_empty()) + .map(|(index, column)| (index, column.name.clone())) + .collect(); + Ok(Some(( + schema.host.audits.clone(), + schema.host.columns.clone(), + audited, + ))) + }) + } + + /// What SQL rules select the model by, plus its cache key inputs when `keyed`. + pub fn sql_rule_inputs(&self, py: Python<'_>, keyed: bool) -> PyResult { + read_input_with_owner(&self.owner, self.index, py, |owner, record| { + let facts: &CompiledModelFacts = + model_facts(owner.facts(), &record.name).ok_or_else(|| { + pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") + })?; + let excluded: Option = match &facts.config.encoding_error { + Some(_) => None, + None => Some( + facts.config.values.get(SQL_ANALYSIS_KEY) + == Some(&serde_json::Value::Bool(false)), + ), + }; + let key = keyed + .then(|| -> PyResult { + Ok(SqlRuleKey { + authored_sql: facts.authored_sql.to_string(), + query_sql: facts.query_sql.to_string(), + contract: compiled_of(record)? + .analysis + .proof + .as_ref() + .map(contract_proof_json), + }) + }) + .transpose()?; + Ok(SqlRuleInputs { + posix_path: facts.relative_path.clone(), + excluded, + key, + }) + }) + } + + /// The model's name, `str(relative_path)` and authored file text. + pub fn file_texts(&self, py: Python<'_>) -> PyResult<(String, String, Arc)> { + self.read(py, |record, py| { + let model_file: &NativeModelFile = record.host.model_file.get(); + Ok(( + model_name(record), + model_file.relative_text(py)?, + Arc::clone(model_file.contents_text()), + )) + }) + } + + /// The name, compiled query and authored file of the model, and whether it calls a function. + pub fn metadata_texts( + &self, + py: Python<'_>, + function_kinds: &[&str], + ) -> PyResult { + self.read(py, |record, _| { + Ok(ModelMetadataTexts { + name: model_name(record), + query_sql: record.query_sql.to_string(), + authored_sql: record.host.model_file.get().contents_text().to_string(), + calls_functions: record + .references + .iter() + .any(|reference| function_kinds.contains(&reference.kind.as_str())), + }) + }) + } + + /// `relative_path.as_posix()` and the compiled query, which compiled artifacts are written from. + pub fn output_texts(&self, py: Python<'_>) -> PyResult<(String, Arc)> { + self.read(py, |record, py| { + Ok(( + record.host.model_file.get().posix_path(py)?, + Arc::clone(&record.query_sql), + )) + }) + } + + /// The model's authored file text and expanded query, from its recorded facts. + pub fn sql_texts(&self, py: Python<'_>) -> PyResult<(String, String)> { + let name: String = self.name(py)?; + let owner = self.owner.bind(py).borrow(); + let facts: &CompiledModelFacts = model_facts(owner.facts(), &name).ok_or_else(|| { + pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") + })?; + Ok((facts.authored_sql.to_string(), facts.query_sql.to_string())) + } + + pub(crate) fn model_file( + &self, + py: Python<'_>, + read: impl FnOnce(&NativeModelFile) -> PyResult, + ) -> PyResult { + self.read(py, |record, _| read(record.host.model_file.get())) + } +} + +#[pymethods] +impl NativeCompiledModel { + #[getter] + pub fn key(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + let compiled: &CompiledRecord = compiled_of(record)?; + compiled + .host + .key + .get_or_try_init(py, || { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + object_key(&classes.compiled, MODEL_RESOURCE, &model_name(record)) + }) + .map(|key| key.clone_ref(py)) + }) + } + + #[getter] + pub fn deps(&self, py: Python<'_>) -> PyResult> { + self.compiled(py, |compiled, py| { + compiled + .host + .deps + .get_or_try_init(py, || { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + let mut keys: Vec> = Vec::with_capacity(compiled.deps.len()); + for (kind, name) in &compiled.deps { + keys.push(object_key(&classes.compiled, kind, name)?); + } + Ok::<_, pyo3::PyErr>(PyTuple::new(py, keys)?.unbind()) + }) + .map(|deps| deps.clone_ref(py)) + }) + } + + #[getter] + pub fn name(&self, py: Python<'_>) -> PyResult { + self.read(py, |record, _| Ok(model_name(record))) + } + + #[getter] + pub fn relative_path(&self, py: Python<'_>) -> PyResult> { + self.model_file(py, |file| file.relative_path(py)) + } + + #[getter] + pub fn query_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.read(py, |record, py| Ok(query_sql_text(py, record))) + } + + #[getter] + pub fn config(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.host.config.clone_ref(py))) + } + + #[getter] + pub fn destination(&self, py: Python<'_>) -> PyResult> { + self.read_destination(py, |destination| { + self.projections + .destination + .get_or_try_init(py, || { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + relation_location(&classes.compiled, destination) + }) + .map(|destination| destination.clone_ref(py)) + }) + } + + #[getter] + pub fn references(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, _| references(&classes, record)) + } + + /// Each reference as `(kind, name, package, call argument count)`. + #[getter] + pub fn reference_rows(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, _| { + Ok(record + .references + .iter() + .cloned() + .map(reference_row_tuple) + .collect()) + }) + } + + /// The model's declared key facts, for deriving its unique keys. + #[getter] + pub fn relation_key_row(&self, py: Python<'_>) -> PyResult { + read_input_with_owner(&self.owner, self.index, py, |owner, record| { + model_relation_key_row(owner, record, py) + }) + } + + /// `(resource type, name)` of the model's key. + #[getter] + pub fn key_row(&self, py: Python<'_>) -> PyResult<(String, String)> { + self.read(py, |record, _| { + Ok((MODEL_RESOURCE.to_owned(), model_name(record))) + }) + } + + /// `(resource type, name)` of each dependency. + #[getter] + pub fn dep_rows(&self, py: Python<'_>) -> PyResult> { + self.compiled(py, |compiled, _| Ok(compiled.deps.clone())) + } + + /// `(database, schema, name)` of the model's destination. + #[getter] + pub fn destination_row(&self, py: Python<'_>) -> PyResult { + self.read_destination(py, |destination| { + Ok(( + destination.database.clone(), + destination.schema.clone(), + destination.name.clone(), + )) + }) + } + + /// Each inferred column as `(name, type, nullability)`, or `None` when none were inferred. + #[getter] + pub fn inferred_rows(&self, py: Python<'_>) -> PyResult>> { + self.compiled(py, |compiled, _| { + let Some(rows) = &compiled.analysis.inferred else { + return Ok(None); + }; + let mut shown: Vec = Vec::with_capacity(rows.len()); + for (name, column_type, nullability) in rows { + let poisoned: bool = self.overlay.poisoned.contains(name); + let column_type: Option = if poisoned { None } else { column_type.clone() }; + shown.push((name.clone(), column_type, nullability.clone())); + } + Ok(Some(shown)) + }) + } + + /// Each declared column of the attached schema as `(name, type)`, or `None` without one. + #[getter] + pub fn schema_column_rows(&self, py: Python<'_>) -> PyResult>> { + let name: String = self.name(py)?; + let owner = self.owner.bind(py).borrow(); + let facts: &CompiledModelFacts = model_facts(owner.facts(), &name).ok_or_else(|| { + pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") + })?; + let Some(schema) = &facts.schema else { + return Ok(None); + }; + Ok(Some( + schema + .columns + .iter() + .map(|column| (column.name.clone(), column.data_type.clone())) + .collect(), + )) + } + + /// Whether the attached schema declares dynamic columns. + #[getter(has_dynamic_columns)] + pub fn dynamic_columns_declared(&self, py: Python<'_>) -> PyResult { + self.read(py, |record, _py| { + Ok(record + .schema + .as_ref() + .is_some_and(|schema| !schema.host.dynamic_columns.specs.is_empty())) + }) + } + + /// Inferred columns, else the attached schema's declared columns, else zero. + #[getter] + pub fn output_column_count(&self, py: Python<'_>) -> PyResult { + self.read(py, |record, _py| { + if let Some(rows) = &compiled_of(record)?.analysis.inferred { + return Ok(rows.len()); + } + Ok(record + .schema + .as_ref() + .map_or(0, |schema| schema.host.columns.specs.len())) + }) + } + + #[getter] + pub fn schema_entry(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, _| schema_entry(&classes, record)) + } + + #[getter] + pub fn inferred_columns(&self, py: Python<'_>) -> PyResult> { + self.compiled(py, |compiled, py| { + self.projections + .inferred_columns + .get_or_try_init(py, || { + let Some(rows) = &compiled.analysis.inferred else { + return Ok::<_, pyo3::PyErr>(py.None()); + }; + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + let mut columns: Vec> = Vec::with_capacity(rows.len()); + for row in rows { + let poisoned_row: InferredRow; + let row = if self.overlay.poisoned.contains(&row.0) { + poisoned_row = (row.0.clone(), None, row.2.clone()); + &poisoned_row + } else { + row + }; + columns.push(self.owner.bind(py).borrow().inferred_column( + py, + &classes.compiled, + row, + )?); + } + Ok(PyTuple::new(py, columns)?.into_any().unbind()) + }) + .map(|columns| columns.clone_ref(py)) + }) + } + + /// The upstream source count of the model's lineage facts, `None` without them. + pub fn lineage_edge_count(&self, py: Python<'_>) -> PyResult> { + self.compiled(py, |compiled, _| Ok(compiled.analysis.lineage_edge_count())) + } + + #[getter] + pub fn fast_lineage_columns(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.compiled(py, |compiled, py| { + compiled.analysis.lineage_object(py, &classes.compiled) + }) + } + + #[getter] + pub fn fast_lineage_has_star(&self, py: Python<'_>) -> PyResult { + self.compiled(py, |compiled, _| Ok(compiled.analysis.has_star)) + } + + #[getter] + pub fn fast_lineage_star_resolved(&self, py: Python<'_>) -> PyResult { + self.compiled(py, |compiled, _| Ok(compiled.analysis.star_resolved)) + } + + #[getter] + pub fn authored_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.model_file(py, |file| Ok(file.contents(py))) + } + + #[getter] + pub fn authored_query_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.model_file(py, |file| Ok(file.query_sql(py))) + } + + #[getter] + pub fn output_column_locations<'py>(&self, py: Python<'py>) -> PyResult> { + self.model_file(py, |file| file.output_column_locations(py)) + } + + #[getter] + pub fn extract_implicit_alias_columns(&self, py: Python<'_>) -> PyResult { + self.model_file(py, |file| Ok(file.extract_implicit_alias_columns())) + } + + #[getter] + pub fn macro_deps<'py>(&self, py: Python<'py>) -> PyResult> { + self.compiled(py, |compiled, py| PyTuple::new(py, &compiled.macro_deps)) + } + + #[getter] + pub fn enum_declarations(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + Ok(record.host.enum_declarations.clone_ref(py)) + }) + } + + #[getter] + pub fn constant_declarations(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + Ok(record.host.constant_declarations.clone_ref(py)) + }) + } + + #[getter] + pub fn enum_columns(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.host.enum_columns.clone_ref(py))) + } + + #[getter] + pub fn binding_diagnostics(&self, py: Python<'_>) -> PyResult> { + if let Some(diagnostics) = &self.overlay.binding_diagnostics { + return Ok(diagnostics.clone_ref(py)); + } + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, py| { + let name: String = model_name(record); + let path: Bound<'_, PyAny> = record.host.model_file.get().path_objects(py)?.1; + compiled_of(record)? + .analysis + .binding_tuple(py, &classes.compiled, (&name, &path)) + }) + } + + #[getter] + pub fn binding_validated(&self, py: Python<'_>) -> PyResult { + self.compiled(py, |compiled, _| Ok(compiled.analysis.binding_validated)) + } + + #[getter] + pub fn dynamic_column_contract(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.compiled(py, |compiled, py| { + compiled.analysis.proof_object(py, &classes.compiled) + }) + } + + #[getter] + pub fn unchecked_output_columns<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + PyFrozenSet::new(py, &self.overlay.poisoned) + } + + #[getter] + pub fn rejected_sql_analysis_opt_out(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(rejected_opt_out(record, py))) + } + + /// This model with its final binding diagnostics. + pub(crate) fn with_binding_diagnostics( + &self, + py: Python<'_>, + diagnostics: Py, + ) -> Self { + self.changed(py, |overlay| { + overlay.binding_diagnostics = Some(diagnostics) + }) + } + + /// This model after semantic type recovery: its bindings and the outputs left unchecked. + pub(crate) fn with_type_recovery( + &self, + py: Python<'_>, + diagnostics: Py, + poisoned: Vec, + ) -> Self { + self.changed(py, |overlay| { + overlay.binding_diagnostics = Some(diagnostics); + overlay.poisoned = poisoned; + }) + } + + /// This model with the adapter's default schema and database filled into its destination. + pub fn with_target_defaults( + &self, + py: Python<'_>, + defaults: (Option, Option), + render: Bound<'_, PyAny>, + ) -> PyResult { + let destination: DestinationFact = self.compiled(py, |compiled, _| { + let current: &DestinationFact = self + .overlay + .destination + .as_ref() + .unwrap_or(&compiled.destination); + defaulted_destination(current, defaults, &render) + })?; + Ok(self.changed(py, |overlay| overlay.destination = Some(destination))) + } +} + +/// One compile's attached inputs, its model inputs projected from the owner. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeInputsView { + pub(crate) owner: Py, + pub(crate) fields: InputsFields, +} + +impl NativeInputsView { + /// The view over `owner` with these input fields. + pub fn new(owner: Py, fields: &Bound<'_, PyDict>) -> PyResult { + Ok(Self { + owner, + fields: InputsFields::from_fields(fields)?, + }) + } +} + +#[pymethods] +impl NativeInputsView { + /// These inputs with `changes` applied; the owner's model inputs are shared. + #[pyo3(signature = (**changes))] + pub(crate) fn with_changes( + &self, + py: Python<'_>, + changes: Option>, + ) -> PyResult { + let current: Bound<'_, PyDict> = self.fields.as_fields(py)?; + if let Some(changes) = changes { + for (name, value) in changes.iter() { + if !current.contains(&name)? { + return Err(PyTypeError::new_err(format!( + "unknown compile input field {name}" + ))); + } + current.set_item(name, value)?; + } + } + Self::new(self.owner.clone_ref(py), ¤t) + } + + /// Every field by name, for an explicit `CompileProjectInputs` projection. + pub(crate) fn fields<'py>(&self, py: Python<'py>) -> PyResult> { + let fields: Bound<'py, PyDict> = self.fields.as_fields(py)?; + fields.set_item("native_project", self.owner.bind(py))?; + Ok(fields) + } + + #[getter] + pub fn run_id(&self) -> String { + self.fields.run_id.clone() + } + + #[getter] + pub fn effective_target_name(&self) -> Option { + self.fields.effective_target_name.clone() + } + + #[getter(no_sql_validation)] + pub fn is_sql_validation_skipped(&self) -> bool { + self.fields.no_sql_validation + } + + #[getter] + pub fn model_inputs(&self, py: Python<'_>) -> Py { + self.fields.model_inputs.clone_ref(py) + } + + #[getter] + pub fn native_project(&self, py: Python<'_>) -> Py { + self.owner.clone_ref(py) + } + + #[getter] + pub fn project_config(&self, py: Python<'_>) -> Py { + self.fields.boundary.project_config.clone_ref(py) + } + + #[getter] + pub fn local_config(&self, py: Python<'_>) -> Py { + self.fields.boundary.local_config.clone_ref(py) + } + + #[getter] + pub fn discovered_inputs(&self, py: Python<'_>) -> Py { + self.fields.boundary.discovered_inputs.clone_ref(py) + } + + #[getter] + pub fn sql_lexical_syntax(&self, py: Python<'_>) -> Py { + self.fields.boundary.sql_lexical_syntax.clone_ref(py) + } + + #[getter] + pub fn effective_target(&self, py: Python<'_>) -> Py { + self.fields.boundary.effective_target.clone_ref(py) + } + + #[getter] + pub fn compile_cache_dir(&self, py: Python<'_>) -> Py { + self.fields.boundary.compile_cache_dir.clone_ref(py) + } + + #[getter] + pub fn effective_connection(&self, py: Python<'_>) -> Py { + self.fields.boundary.effective_connection.clone_ref(py) + } + + #[getter] + pub fn effective_vars(&self, py: Python<'_>) -> Py { + self.fields.boundary.effective_vars.clone_ref(py) + } + + #[getter] + pub fn macro_context(&self, py: Python<'_>) -> Py { + self.fields.boundary.macro_context.clone_ref(py) + } + + #[getter] + pub fn loaded_macros(&self, py: Python<'_>) -> Py { + self.fields.boundary.loaded_macros.clone_ref(py) + } + + #[getter] + pub fn public_enums(&self, py: Python<'_>) -> Py { + self.fields.boundary.public_enums.clone_ref(py) + } + + #[getter] + pub fn public_constants(&self, py: Python<'_>) -> Py { + self.fields.boundary.public_constants.clone_ref(py) + } + + #[getter] + pub fn external_sql_reference_resolver(&self, py: Python<'_>) -> Py { + self.fields + .boundary + .external_sql_reference_resolver + .clone_ref(py) + } + + #[getter] + pub fn effective_settings(&self, py: Python<'_>) -> Py { + self.fields.compiler.effective_settings.clone_ref(py) + } + + #[getter] + pub fn seed_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.seed_inputs.clone_ref(py) + } + + #[getter] + pub fn source_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.source_inputs.clone_ref(py) + } + + #[getter] + pub fn sql_function_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.sql_function_inputs.clone_ref(py) + } + + #[getter] + pub fn test_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.test_inputs.clone_ref(py) + } + + #[getter] + pub fn scenario_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.scenario_inputs.clone_ref(py) + } + + #[getter] + pub fn audit_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.audit_inputs.clone_ref(py) + } + + #[getter] + pub fn diagnostics(&self, py: Python<'_>) -> Py { + self.fields.compiler.diagnostics.clone_ref(py) + } + + #[getter] + pub fn scope_index(&self, py: Python<'_>) -> Py { + self.fields.compiler.scope_index.clone_ref(py) + } + + #[getter] + pub fn declaration_scope(&self, py: Python<'_>) -> Py { + self.fields.compiler.declaration_scope.clone_ref(py) + } +} + +/// The Python classes a model input's projected attributes are built with. +#[derive(Debug, FromPyObject)] +#[pyo3(from_item_all)] +pub struct InputViewClasses<'py> { + pub compiled: CompiledViewClasses<'py>, + pub(crate) sql_reference: Bound<'py, PyAny>, + pub(crate) reference_kind: Bound<'py, PyAny>, + pub(crate) schema_model_entry: Bound<'py, PyAny>, + pub(crate) sql_expansion: Bound<'py, PyAny>, + pub(crate) expansion_span: Bound<'py, PyAny>, +} + +/// A model schema's header columns and specs, which carry user-authored audit arguments. +pub struct SchemaHandles { + pub columns: Arc, + pub audits: Arc, + pub dynamic_columns: Arc, +} + +/// A model input's user values and the attribute projections its views build. +pub struct ModelInputHandles { + /// The discovered model file, owned by the discovery inputs. + pub model_file: Py, + /// The resolved config, whose values hold user values. + pub config: Py, + pub rejected_opt_out: Option>, + /// The model's own enums and constants, whose values are user values. + pub enum_declarations: Py, + pub constant_declarations: Py, + pub enum_columns: Py, + /// Usage records, the identities the scope index is built from. + pub macro_usages: Py, + pub declaration_usages: Py, + /// Attribute projections built on first access, shared by every view of the model. + pub projections: InputProjections, +} + +/// A model input's Python attribute projections, each built once on first access. +pub struct InputProjections { + pub(crate) query_sql: PyOnceLock>, + pub(crate) references: PyOnceLock>, + pub(crate) schema_entry: PyOnceLock>, + pub(crate) sql_expansion: PyOnceLock>, + pub(crate) lint_expansion: PyOnceLock>, +} + +impl std::fmt::Debug for InputProjections { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("InputProjections") + .finish_non_exhaustive() + } +} + +impl Default for InputProjections { + fn default() -> Self { + Self { + query_sql: PyOnceLock::new(), + references: PyOnceLock::new(), + schema_entry: PyOnceLock::new(), + sql_expansion: PyOnceLock::new(), + lint_expansion: PyOnceLock::new(), + } + } +} + +/// One model's compile input, projected from the owner on demand. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeModelInput { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeModelInput { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } + + pub fn read<'py, T>( + &self, + py: Python<'py>, + read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, + ) -> PyResult { + read_input(&self.owner, self.index, py, read) + } + + /// Read the input record together with the project that owns it. + pub fn read_with_owner<'py, T>( + &self, + py: Python<'py>, + read: impl FnOnce(&NativeCompiledProject, &ModelInputRecord) -> PyResult, + ) -> PyResult { + read_input_with_owner(&self.owner, self.index, py, read) + } +} + +#[pymethods] +impl NativeModelInput { + #[getter] + pub fn model_file(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + Ok(record.host.model_file.clone_ref(py).into_any()) + }) + } + + #[getter] + pub fn config(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.host.config.clone_ref(py))) + } + + #[getter] + pub fn query_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.read(py, |record, py| Ok(query_sql_text(py, record))) + } + + #[getter] + pub fn macro_source_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.read(py, |record, py| { + Ok(PyString::new(py, &record.macro_source_sql)) + }) + } + + #[getter] + pub fn references(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, _| references(&classes, record)) + } + + #[getter] + pub fn schema_entry(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, _| schema_entry(&classes, record)) + } + + /// Each reference as `(kind, name, package, call argument count)`. + #[getter] + pub fn reference_rows(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, _| { + Ok(record + .references + .iter() + .cloned() + .map(reference_row_tuple) + .collect()) + }) + } + + /// The model's declared key facts, for deriving its unique keys. + #[getter] + pub fn relation_key_row(&self, py: Python<'_>) -> PyResult { + self.read_with_owner(py, |owner, record| { + model_relation_key_row(owner, record, py) + }) + } + + /// The name of the schema entry attached to the model, or `None`. + #[getter] + pub fn attached_schema_name(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, _| Ok(attached_schema_name(record))) + } + + #[getter] + pub fn schema_file(&self, py: Python<'_>) -> Py { + py.None() + } + + #[getter] + pub fn sql_validation_enabled(&self, py: Python<'_>) -> PyResult { + self.read(py, |record, _| Ok(record.sql_validation_enabled)) + } + + #[getter] + pub fn rejected_sql_analysis_opt_out(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(rejected_opt_out(record, py))) + } + + #[getter] + pub fn enum_declarations(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + Ok(record.host.enum_declarations.clone_ref(py)) + }) + } + + #[getter] + pub fn constant_declarations(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + Ok(record.host.constant_declarations.clone_ref(py)) + }) + } + + #[getter] + pub fn enum_columns(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.host.enum_columns.clone_ref(py))) + } + + #[getter] + pub fn macro_deps<'py>(&self, py: Python<'py>) -> PyResult> { + self.read(py, |record, py| PyTuple::new(py, &record.macro_deps)) + } + + #[getter] + pub fn macro_usages(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.host.macro_usages.clone_ref(py))) + } + + #[getter] + pub fn declaration_usages(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + Ok(record.host.declaration_usages.clone_ref(py)) + }) + } + + /// Record what assembly finished for the model; return its compiled model view. + pub(crate) fn compiled_model( + &self, + py: Python<'_>, + arguments: CompiledArguments, + ) -> PyResult { + let mut owner = self.owner.bind(py).borrow_mut(); + let record: &mut ModelInputRecord = owner.input_mut(self.index)?; + let name: String = record.name.clone(); + record.compiled = Some(compiled_record(py, record, &name, arguments)?); + Ok(NativeCompiledModel::new( + self.owner.clone_ref(py), + self.index, + )) + } + + /// The authored-to-expanded passes, when interpolation left the authored query unchanged. + #[getter] + pub fn sql_expansion(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, _| authored_expansion(&classes, record)) + } + + /// The whole expansion, variables included, which early SQL lint reuses. + #[getter] + pub fn lint_expansion(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, py| { + record + .host + .projections + .lint_expansion + .get_or_try_init(py, || { + let expansion: &ExpansionRecord = &record.expansion; + expansion_object( + &classes, + expansion, + &[ + &expansion.interpolation_spans, + &expansion.declaration_spans, + &expansion.macro_spans, + ], + ) + }) + .map(|expansion| expansion.clone_ref(py)) + }) + } +} + +/// A model input's attached schema: its name and dynamic column families. +#[derive(Debug)] +pub struct AttachedSchema { + pub name: String, + pub dynamic_columns: Vec, + pub type_enforcement: bool, +} + +/// One lineage output with its transform kind and confidence values, and its sources. +#[derive(Debug)] +pub struct LineageFactRow { + pub output_column: String, + pub transform: String, + pub confidence: String, + /// `(resource type, resource, column)` of each source. + pub sources: Vec<(String, String, String)>, +} + +/// The extension's host: Python handles and the views built from them. +#[derive(Debug)] +pub struct PyProjectHost; + +impl ProjectHost for PyProjectHost { + type ModelInput = ModelInputHandles; + type Schema = SchemaHandles; + type CompiledModel = CompiledProjections; + type Analysis = AnalysisProjections; + type AttachedSource = SourceHandles; + type Source = SourceProjections; + type AttachedFunction = FunctionHandles; + type Function = FunctionProjections; + type AttachedAudit = AuditHandles; + type Audit = AuditProjections; + type Seed = SeedHandles; +} + +/// One compiled project, its models projected from the owner. +#[pyclass(module = "sqlbuild._native", frozen)] +pub struct NativeProjectView { + pub(crate) owner: Py, + pub(crate) fields: ProjectFields, + pub(crate) lineage_graph: PyOnceLock>, + pub(crate) sql_expansions: PyOnceLock>, +} + +impl std::fmt::Debug for NativeProjectView { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("NativeProjectView") + .finish_non_exhaustive() + } +} + +impl NativeProjectView { + /// The view over `owner` with these project fields. + pub fn new(owner: Py, fields: &Bound<'_, PyDict>) -> PyResult { + Ok(Self::with_fields( + owner, + ProjectFields::from_fields(fields)?, + )) + } + + /// The project field custom-rule facts and rules runs read by `name`, or `None` for any other. + pub fn fact_field(&self, py: Python<'_>, name: &str) -> Option> { + let field: &Py = match name { + "settings" => &self.fields.compiler.settings, + "sql_scenarios" => &self.fields.compiler.sql_scenarios, + "scope_index" => &self.fields.compiler.scope_index, + "diagnostics" => &self.fields.compiler.diagnostics, + "sources" => &self.fields.compiler.sources, + "seeds" => &self.fields.compiler.seeds, + "functions" => &self.fields.compiler.functions, + "audits" => &self.fields.compiler.audits, + "sql_tests" => &self.fields.compiler.sql_tests, + "public_enums" => &self.fields.boundary.public_enums, + "public_constants" => &self.fields.boundary.public_constants, + "models" => return Some(self.fields.models.clone_ref(py).into_any()), + _ => return None, + }; + Some(field.clone_ref(py)) + } + + /// The compiled project this view projects. + pub fn owner(&self) -> &Py { + &self.owner + } + + pub(crate) fn with_fields(owner: Py, fields: ProjectFields) -> Self { + Self { + owner, + fields, + lineage_graph: PyOnceLock::new(), + sql_expansions: PyOnceLock::new(), + } + } +} + +#[pymethods] +impl NativeProjectView { + /// This project with `changes` applied; the owner's models are shared. + #[pyo3(signature = (**changes))] + pub(crate) fn with_changes( + &self, + py: Python<'_>, + changes: Option>, + ) -> PyResult { + let fields: ProjectFields = match changes { + Some(changes) => self.fields.changed(py, &changes)?, + None => self.fields.changed(py, &PyDict::new(py))?, + }; + Ok(Self::with_fields(self.owner.clone_ref(py), fields)) + } + + /// Every field by name, for an explicit `CompiledProject` projection. + #[pyo3(signature = (include_sql_expansions = true))] + pub fn fields<'py>( + &self, + py: Python<'py>, + include_sql_expansions: bool, + ) -> PyResult> { + let fields: Bound<'py, PyDict> = self.fields.as_fields(py)?; + fields.set_item( + "sql_expansions", + if include_sql_expansions { + self.sql_expansions(py)? + } else { + PyDict::new(py).unbind() + }, + )?; + fields.set_item("native_project", self.owner.bind(py))?; + Ok(fields) + } + + #[getter] + pub fn run_id(&self) -> String { + self.fields.run_id.clone() + } + + #[getter] + pub fn effective_target_name(&self) -> Option { + self.fields.effective_target_name.clone() + } + + #[getter] + pub fn effective_target_database(&self) -> Option { + self.fields.effective_target_database.clone() + } + + #[getter] + pub fn effective_target_schema(&self) -> Option { + self.fields.effective_target_schema.clone() + } + + #[getter] + pub fn sql_analysis_dialect(&self) -> Option { + self.fields.sql_analysis_dialect.clone() + } + + #[getter(enforce_explicit_references)] + pub fn is_explicit_reference_enforced(&self) -> bool { + self.fields.enforce_explicit_references + } + + #[getter] + pub fn models(&self, py: Python<'_>) -> Py { + self.fields.models.clone_ref(py) + } + + #[getter] + pub fn native_project(&self, py: Python<'_>) -> Py { + self.owner.clone_ref(py) + } + + #[getter] + pub fn effective_connection(&self, py: Python<'_>) -> Py { + self.fields.boundary.effective_connection.clone_ref(py) + } + + #[getter] + pub fn effective_vars(&self, py: Python<'_>) -> Py { + self.fields.boundary.effective_vars.clone_ref(py) + } + + #[getter] + pub fn binding_catalog(&self, py: Python<'_>) -> Py { + self.fields.boundary.binding_catalog.clone_ref(py) + } + + #[getter] + pub fn sql_lexical_syntax(&self, py: Python<'_>) -> Py { + self.fields.boundary.sql_lexical_syntax.clone_ref(py) + } + + #[getter] + pub fn compile_cache_dir(&self, py: Python<'_>) -> Py { + self.fields.boundary.compile_cache_dir.clone_ref(py) + } + + #[getter] + pub fn loader_functions(&self, py: Python<'_>) -> Py { + self.fields.boundary.loader_functions.clone_ref(py) + } + + #[getter] + pub fn python_execution_inputs(&self) -> BTreeMap { + self.fields.boundary.python_execution_inputs.clone() + } + + #[getter] + pub fn hook_functions(&self, py: Python<'_>) -> Py { + self.fields.boundary.hook_functions.clone_ref(py) + } + + #[getter] + pub fn sql_hook_files(&self, py: Python<'_>) -> Py { + self.fields.boundary.sql_hook_files.clone_ref(py) + } + + #[getter] + pub fn materialization_files(&self, py: Python<'_>) -> Py { + self.fields.boundary.materialization_files.clone_ref(py) + } + + #[getter] + pub fn public_enums(&self, py: Python<'_>) -> Py { + self.fields.boundary.public_enums.clone_ref(py) + } + + #[getter] + pub fn public_constants(&self, py: Python<'_>) -> Py { + self.fields.boundary.public_constants.clone_ref(py) + } + + #[getter] + pub fn loaded_macros(&self, py: Python<'_>) -> Py { + self.fields.boundary.loaded_macros.clone_ref(py) + } + + #[getter] + pub fn external_sql_reference_resolver(&self, py: Python<'_>) -> Py { + self.fields + .boundary + .external_sql_reference_resolver + .clone_ref(py) + } + + #[getter] + pub fn native_session(&self, py: Python<'_>) -> Py { + self.fields.boundary.native_session.clone_ref(py) + } + + #[getter] + pub fn settings(&self, py: Python<'_>) -> Py { + self.fields.compiler.settings.clone_ref(py) + } + + #[getter] + pub fn scenario(&self, py: Python<'_>) -> Py { + self.fields.compiler.scenario.clone_ref(py) + } + + #[getter] + pub fn sources(&self, py: Python<'_>) -> Py { + self.fields.compiler.sources.clone_ref(py) + } + + #[getter] + pub fn seeds(&self, py: Python<'_>) -> Py { + self.fields.compiler.seeds.clone_ref(py) + } + + #[getter] + pub fn functions(&self, py: Python<'_>) -> Py { + self.fields.compiler.functions.clone_ref(py) + } + + #[getter] + pub fn audits(&self, py: Python<'_>) -> Py { + self.fields.compiler.audits.clone_ref(py) + } + + #[getter] + pub fn sql_tests(&self, py: Python<'_>) -> Py { + self.fields.compiler.sql_tests.clone_ref(py) + } + + #[getter] + pub fn sql_scenarios(&self, py: Python<'_>) -> Py { + self.fields.compiler.sql_scenarios.clone_ref(py) + } + + #[getter] + pub fn diagnostics(&self, py: Python<'_>) -> Py { + self.fields.compiler.diagnostics.clone_ref(py) + } + + #[getter] + pub fn unmatched_literal_sql_relations(&self, py: Python<'_>) -> Py { + self.fields + .compiler + .unmatched_literal_sql_relations + .clone_ref(py) + } + + #[getter] + pub fn scope_index(&self, py: Python<'_>) -> Py { + self.fields.compiler.scope_index.clone_ref(py) + } + + /// Each model's authored-to-expanded passes by file path, where interpolation kept the query. + #[getter] + pub fn sql_expansions(&self, py: Python<'_>) -> PyResult> { + self.sql_expansions + .get_or_try_init(py, || { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + let owner = self.owner.bind(py).borrow(); + let expansions: Bound<'_, PyDict> = PyDict::new(py); + for record in owner.inputs() { + let expansion: Py = authored_expansion(&classes, record)?; + if !expansion.is_none(py) { + let path: Bound<'_, PyAny> = + record.host.model_file.get().path_objects(py)?.0; + expansions.set_item(path, expansion)?; + } + } + Ok::<_, pyo3::PyErr>(expansions.unbind()) + }) + .map(|expansions| expansions.clone_ref(py)) + } + + /// The lineage edges, tags, model folders and names, indexed natively once per project. + #[getter] + pub fn lineage_graph(&self, py: Python<'_>) -> PyResult> { + self.lineage_graph + .get_or_try_init(py, || { + let mut rows: Vec = Vec::new(); + for model in self.fields.models.bind(py).iter() { + let model: Bound<'_, NativeCompiledModel> = model.downcast_into()?; + rows.push(model_graph_row(model.get(), py)?); + } + let compiler: &CompilerHandles = &self.fields.compiler; + for source in compiler.sources.bind(py).try_iter()? { + let source: Bound<'_, PyAny> = source?; + rows.push(match source.cast::() { + Ok(native) => native.get().graph_row(py)?, + Err(_) => resource_row(&source, None)?, + }); + } + for seed in compiler.seeds.bind(py).try_iter()? { + let seed: Bound<'_, PyAny> = seed?; + rows.push(match seed.cast::() { + Ok(native) => native.get().graph_row(py)?, + Err(_) => { + let tags: Bound<'_, PyAny> = native_field( + &native_field(&seed, SCHEMA_ENTRY_ATTRIBUTE)?, + TAGS_ATTRIBUTE, + )?; + resource_row(&seed, Some(&tags))? + } + }); + } + for function in compiler.functions.bind(py).try_iter()? { + let function: Bound<'_, PyAny> = function?; + rows.push(match function.cast::() { + Ok(native) => native.get().graph_row(py)?, + Err(_) => { + let tags: Bound<'_, PyAny> = native_field(&function, TAGS_ATTRIBUTE)?; + resource_row(&function, Some(&tags))? + } + }); + } + self.owner.bind(py).borrow_mut().record_graph_build(); + count_stage(py, LINEAGE_GRAPH_BUILDS, 1)?; + Py::new(py, project_graph(rows)?) + }) + .map(|graph| graph.clone_ref(py)) + } +} + +/// `(kind, name, materialized, mode, strategy, cursor, unique key, columns, audits)`. +#[derive(Debug)] +pub struct RelationKeyRow { + pub kind: String, + pub name: String, + pub materialized: String, + pub mode: String, + pub strategy: String, + pub has_cursor: bool, + pub unique_key: Option>, + pub columns: Vec<(String, Vec, bool)>, + pub audits: Vec<(String, Option)>, +} + +impl<'py> pyo3::FromPyObject<'py> for RelationKeyRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (kind, name, materialized, mode, strategy, has_cursor, unique_key, columns, audits): RelationKeyTuple = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + kind, + name, + materialized, + mode, + strategy, + has_cursor, + unique_key, + columns, + audits, + }) + } +} + +impl<'py> pyo3::IntoPyObject<'py> for RelationKeyRow { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject( + ( + self.kind, + self.name, + self.materialized, + self.mode, + self.strategy, + self.has_cursor, + self.unique_key, + self.columns, + self.audits, + ), + py, + ) + } +} diff --git a/crates/sqlbuild-python-project/src/compiled_views/types.rs b/crates/sqlbuild-python-project/src/compiled_views/types.rs new file mode 100644 index 0000000000..0bb954a875 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_views/types.rs @@ -0,0 +1,32 @@ +//! Public traits and type aliases of compiled views. + +use crate::compiled_views::models::PyProjectHost; +use sqlbuild_project::models as project_models; + +/// `(name, type)` of a declared column. +pub type DeclaredRow = (String, Option); + +/// `(output column, [(resource name, column name)])` of one lineage output. +pub type LineageInput = (String, Vec<(String, String)>); + +/// A model's attached header schema; the host keeps its columns, audits and dynamic columns. +pub type SchemaRecord = project_models::SchemaRecord; + +/// What the model loop finished for one model beyond its facts. +pub type ModelInputRecord = project_models::ModelInputRecord; + +/// Declared keys by `kind:name` identity, in identity order. +pub type RelationKeys = Vec<(String, Vec>)>; + +/// The tuple Python sends for a `RelationKeyRow`, in field order. +pub type RelationKeyTuple = ( + String, + String, + String, + String, + String, + bool, + Option>, + Vec<(String, Vec, bool)>, + Vec<(String, Option)>, +); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs b/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs similarity index 81% rename from crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs rename to crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs index 361d2c5738..861ac32076 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs +++ b/crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs @@ -2,11 +2,9 @@ use std::collections::HashMap; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction}; use serde_json::Value; use sqlbuild_analysis::compiled_project::main::project_models::project_models; use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; @@ -18,11 +16,11 @@ use sqlbuild_analysis::contracts::models::{ use sqlbuild_core::constants::STAGED_PROMOTION_MODE; use sqlbuild_sqltext::compiler::main::model_output_columns::model_output_columns; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::model_reads::enum_columns; -use crate::bindings::_helpers::compiled_views::records::compiled_project_facts; -use crate::bindings::_helpers::contracts::model_contracts::{ - ConflictRow, DiagnosticRow, evaluate_project_model_contracts, project_promotion_conflicts, +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::main::compiled_models::enum_columns::enum_columns; +use crate::compiled_views::main::project::compiled_project_facts::compiled_project_facts; +use crate::contracts::_helpers::model_contracts::{ + DiagnosticRow, evaluate_project_model_contracts, }; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::record_class::record_class; @@ -31,14 +29,14 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::sqltext::main::location_rows::location_rows; use sqlbuild_python_front::sqltext::models::ColumnLocation; -const PATHLIB_MODULE: &str = "pathlib"; +pub(crate) const PATHLIB_MODULE: &str = "pathlib"; const GENERIC_DIALECT: &str = "generic"; const OUTPUT_LABEL: &str = "output"; -const CONTRACT_PHASE: &str = "contract"; +pub(crate) const CONTRACT_PHASE: &str = "contract"; const MODEL_RESOURCE: &str = "model"; const SQL_ANALYSIS_KEY: &str = "sql_analysis"; -fn keyword_call<'py>( +pub(crate) fn keyword_call<'py>( callable: &Bound<'py, PyAny>, pairs: Vec<(&str, Bound<'py, PyAny>)>, ) -> PyResult> { @@ -50,7 +48,7 @@ fn keyword_call<'py>( } /// The compile's retained native project, read through `_native.compiled_project_facts`. -fn native_project<'py>(project: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn native_project<'py>(project: &Bound<'py, PyAny>) -> PyResult> { Ok(compiled_project_facts(project)?.into_any()) } @@ -282,7 +280,7 @@ fn enum_literal_diagnostic<'py>( ) } -fn model_resource_type(py: Python<'_>) -> PyResult> { +pub(crate) fn model_resource_type(py: Python<'_>) -> PyResult> { enum_from_value(py, RecordId::CompiledResourceType, MODEL_RESOURCE) } @@ -348,59 +346,3 @@ pub(crate) fn evaluate_model_contracts<'py>( vec![("diagnostics", PyTuple::new(py, diagnostics)?.into_any())], ) } - -/// One K011 error per enforced-contract table model under immediate promotion. -#[pyfunction] -#[pyo3(signature = (*, project, adapter_default, settings_file))] -pub(crate) fn promotion_conflict_diagnostics<'py>( - py: Python<'py>, - project: &Bound<'py, PyAny>, - adapter_default: &Bound<'py, PyAny>, - settings_file: String, -) -> PyResult> { - let explicit_mode: Option = - native_field(&native_field(project, "settings")?, "table_promotion_mode")?.extract()?; - let native: Bound<'py, PyAny> = native_project(project)?; - let native: PyRef<'py, NativeCompiledProject> = native.extract()?; - let conflicts: Vec = project_promotion_conflicts( - py, - ( - explicit_mode, - adapter_default.str()?.extract()?, - settings_file, - ), - &native, - )?; - let diagnostic_type: Bound<'py, PyAny> = record_class(py, RecordId::CompilerDiagnostic)?; - let path_type: Bound<'py, PyAny> = py.import(PATHLIB_MODULE)?.getattr("Path")?; - let mut diagnostics: Vec> = Vec::with_capacity(conflicts.len()); - for ConflictRow { - name, - relative_path, - code, - message, - help, - } in conflicts - { - diagnostics.push(keyword_call( - &diagnostic_type, - vec![ - ("phase", CONTRACT_PHASE.into_pyobject(py)?.into_any()), - ("severity", "error".into_pyobject(py)?.into_any()), - ("code", code.into_pyobject(py)?.into_any()), - ("message", message.into_pyobject(py)?.into_any()), - ("resource_type", model_resource_type(py)?), - ("resource_name", name.into_pyobject(py)?.into_any()), - ("path", path_type.call1((relative_path,))?), - ("help", help.into_pyobject(py)?.into_any()), - ], - )?); - } - PyTuple::new(py, diagnostics) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(evaluate_model_contracts, module)?)?; - module.add_function(wrap_pyfunction!(promotion_conflict_diagnostics, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/exception_types.rs b/crates/sqlbuild-python-project/src/contracts/_helpers/exception_types.rs similarity index 51% rename from crates/sqlbuild-python/src/bindings/_helpers/contracts/exception_types.rs rename to crates/sqlbuild-python-project/src/contracts/_helpers/exception_types.rs index 5e24f800c2..f3cb42f2e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/exception_types.rs +++ b/crates/sqlbuild-python-project/src/contracts/_helpers/exception_types.rs @@ -2,7 +2,6 @@ use pyo3::create_exception; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; create_exception!( sqlbuild._native, @@ -10,10 +9,3 @@ create_exception!( PyValueError, "An authored value cannot be represented by the typed SQL value domain." ); - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add( - "SqlValueValidationError", - module.py().get_type::(), - ) -} diff --git a/crates/sqlbuild-python-project/src/contracts/_helpers/mod.rs b/crates/sqlbuild-python-project/src/contracts/_helpers/mod.rs new file mode 100644 index 0000000000..7d9a72637e --- /dev/null +++ b/crates/sqlbuild-python-project/src/contracts/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod contract_diagnostics; +pub(crate) mod exception_types; +pub(crate) mod model_contracts; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs b/crates/sqlbuild-python-project/src/contracts/_helpers/model_contracts.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs rename to crates/sqlbuild-python-project/src/contracts/_helpers/model_contracts.rs index e3f1ba85ca..d334f5b780 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs +++ b/crates/sqlbuild-python-project/src/contracts/_helpers/model_contracts.rs @@ -16,7 +16,7 @@ use sqlbuild_analysis::contracts::models::{ use sqlbuild_analysis::sql_macros::main::untyped_call_outputs::untyped_call_outputs; use sqlbuild_analysis::type_system::errors::TypeNormalizationError; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::compiled_project::models::NativeCompiledProject; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::type_system::main::normalization_error::normalization_error; use sqlbuild_python_base::types::CompilerDetach; diff --git a/crates/sqlbuild-python-project/src/contracts/main/evaluate_model_contracts.rs b/crates/sqlbuild-python-project/src/contracts/main/evaluate_model_contracts.rs new file mode 100644 index 0000000000..4b3d34f96e --- /dev/null +++ b/crates/sqlbuild-python-project/src/contracts/main/evaluate_model_contracts.rs @@ -0,0 +1,15 @@ +//! Every model's column contract diagnostics in model order; no build is staged without `adapter_default`. + +use crate::contracts::_helpers::contract_diagnostics; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; + +/// Every model's column contract diagnostics in model order; no build is staged without `adapter_default`. +#[inline] +pub fn evaluate_model_contracts<'py>( + py: Python<'py>, + project: &Bound<'py, PyAny>, + dialect: Option<&Bound<'py, PyAny>>, + adapter_default: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + contract_diagnostics::evaluate_model_contracts(py, project, dialect, adapter_default) +} diff --git a/crates/sqlbuild-python-project/src/contracts/main/mod.rs b/crates/sqlbuild-python-project/src/contracts/main/mod.rs new file mode 100644 index 0000000000..37e04c6f41 --- /dev/null +++ b/crates/sqlbuild-python-project/src/contracts/main/mod.rs @@ -0,0 +1,4 @@ +pub mod evaluate_model_contracts; +pub mod promotion_conflict_diagnostics; +pub mod register_contract_diagnostics; +pub mod register_exception_types; diff --git a/crates/sqlbuild-python-project/src/contracts/main/promotion_conflict_diagnostics.rs b/crates/sqlbuild-python-project/src/contracts/main/promotion_conflict_diagnostics.rs new file mode 100644 index 0000000000..131ddfada9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/contracts/main/promotion_conflict_diagnostics.rs @@ -0,0 +1,65 @@ +//! One K011 error per enforced-contract table model under immediate promotion. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::contracts::_helpers::contract_diagnostics::CONTRACT_PHASE; +use crate::contracts::_helpers::contract_diagnostics::PATHLIB_MODULE; +use crate::contracts::_helpers::contract_diagnostics::keyword_call; +use crate::contracts::_helpers::contract_diagnostics::model_resource_type; +use crate::contracts::_helpers::contract_diagnostics::native_project; +use crate::contracts::_helpers::model_contracts::{ConflictRow, project_promotion_conflicts}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; +use pyo3::types::PyTuple; +use pyo3::{IntoPyObject, pyfunction}; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +/// One K011 error per enforced-contract table model under immediate promotion. +#[pyfunction] +#[pyo3(signature = (*, project, adapter_default, settings_file))] +pub fn promotion_conflict_diagnostics<'py>( + py: Python<'py>, + project: &Bound<'py, PyAny>, + adapter_default: &Bound<'py, PyAny>, + settings_file: String, +) -> PyResult> { + let explicit_mode: Option = + native_field(&native_field(project, "settings")?, "table_promotion_mode")?.extract()?; + let native: Bound<'py, PyAny> = native_project(project)?; + let native: PyRef<'py, NativeCompiledProject> = native.extract()?; + let conflicts: Vec = project_promotion_conflicts( + py, + ( + explicit_mode, + adapter_default.str()?.extract()?, + settings_file, + ), + &native, + )?; + let diagnostic_type: Bound<'py, PyAny> = record_class(py, RecordId::CompilerDiagnostic)?; + let path_type: Bound<'py, PyAny> = py.import(PATHLIB_MODULE)?.getattr("Path")?; + let mut diagnostics: Vec> = Vec::with_capacity(conflicts.len()); + for ConflictRow { + name, + relative_path, + code, + message, + help, + } in conflicts + { + diagnostics.push(keyword_call( + &diagnostic_type, + vec![ + ("phase", CONTRACT_PHASE.into_pyobject(py)?.into_any()), + ("severity", "error".into_pyobject(py)?.into_any()), + ("code", code.into_pyobject(py)?.into_any()), + ("message", message.into_pyobject(py)?.into_any()), + ("resource_type", model_resource_type(py)?), + ("resource_name", name.into_pyobject(py)?.into_any()), + ("path", path_type.call1((relative_path,))?), + ("help", help.into_pyobject(py)?.into_any()), + ], + )?); + } + PyTuple::new(py, diagnostics) +} diff --git a/crates/sqlbuild-python-project/src/contracts/main/register_contract_diagnostics.rs b/crates/sqlbuild-python-project/src/contracts/main/register_contract_diagnostics.rs new file mode 100644 index 0000000000..e98b71d02d --- /dev/null +++ b/crates/sqlbuild-python-project/src/contracts/main/register_contract_diagnostics.rs @@ -0,0 +1,13 @@ +//! Register the contract diagnostics functions and classes. + +use crate::contracts::_helpers::contract_diagnostics::evaluate_model_contracts; +use crate::contracts::main::promotion_conflict_diagnostics::promotion_conflict_diagnostics; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the contract diagnostics functions and classes. +pub fn register_contract_diagnostics(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(evaluate_model_contracts, module)?)?; + module.add_function(wrap_pyfunction!(promotion_conflict_diagnostics, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/contracts/main/register_exception_types.rs b/crates/sqlbuild-python-project/src/contracts/main/register_exception_types.rs new file mode 100644 index 0000000000..0b0e25703e --- /dev/null +++ b/crates/sqlbuild-python-project/src/contracts/main/register_exception_types.rs @@ -0,0 +1,12 @@ +//! Register the exception types functions and classes. + +use crate::contracts::_helpers::exception_types::SqlValueValidationError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the exception types functions and classes. +pub fn register_exception_types(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add( + "SqlValueValidationError", + module.py().get_type::(), + ) +} diff --git a/crates/sqlbuild-python-project/src/contracts/mod.rs b/crates/sqlbuild-python-project/src/contracts/mod.rs new file mode 100644 index 0000000000..c1664aaf89 --- /dev/null +++ b/crates/sqlbuild-python-project/src/contracts/mod.rs @@ -0,0 +1,4 @@ +//! Model contracts, their diagnostics and exception types. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs b/crates/sqlbuild-python-project/src/fingerprints/_helpers/latest_rows.rs similarity index 59% rename from crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs rename to crates/sqlbuild-python-project/src/fingerprints/_helpers/latest_rows.rs index 475ff3013d..cc35719d0b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/_helpers/latest_rows.rs @@ -1,30 +1,26 @@ //! Latest fingerprint rows decoded into `Fingerprint` models and indexed into a `FingerprintSet`. +use crate::fingerprints::main::row_fingerprint::row_fingerprint; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::ffi::CString; use pyo3::exceptions::{PyUnicodeDecodeError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyBytes, PyDateTime, PyDict, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBytes, PyDict, PyString}; use sqlbuild_analysis::fingerprint_storage::errors::StorageDecodeError; use sqlbuild_analysis::fingerprint_storage::main::decode_storage::decode_storage; use sqlbuild_analysis::fingerprint_storage::main::latest_index::latest_index; -use sqlbuild_analysis::fingerprint_storage::main::selected_positions::selected_positions; use sqlbuild_analysis::fingerprint_storage::models::{FingerprintIdentity, LatestIndex}; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; const DATETIME_MODULE: &str = "datetime"; const BINASCII_MODULE: &str = "binascii"; const BASE64_MODULE: &str = "base64"; -const TIMESTAMP_COLUMN: usize = 11; +pub(crate) const TIMESTAMP_COLUMN: usize = 11; /// `FingerprintInputError(message)`. pub(crate) fn fingerprint_input_error(py: Python<'_>, message: &str) -> PyErr { @@ -55,39 +51,7 @@ pub(crate) fn fingerprint_set_from_rows<'py>( indexed_set(py, schema, &fingerprints) } -/// `select_latest_fingerprints`: an unfiltered latest read narrowed as a filtered read would be. -#[pyfunction] -#[pyo3(signature = (*, fingerprint_set, node_names, filtered_node_types = Vec::new()))] -pub(crate) fn select_latest_fingerprints<'py>( - py: Python<'py>, - fingerprint_set: &Bound<'py, PyAny>, - node_names: Vec, - filtered_node_types: Vec, -) -> PyResult> { - let schema: String = native_field(fingerprint_set, "schema")?.extract()?; - if node_names.is_empty() && filtered_node_types.is_empty() { - return indexed_set(py, &schema, &[]); - } - let by_identity: Bound<'py, PyAny> = native_field(fingerprint_set, "fingerprints_by_identity")?; - let latest: Vec> = if by_identity.is_truthy()? { - mapping_values(&by_identity)? - .try_iter()? - .collect::>()? - } else { - Vec::new() - }; - let identities: Vec<(String, String)> = - latest.iter().map(identity_of).collect::>()?; - let positions: Vec = - selected_positions(&borrowed(&identities), &node_names, &filtered_node_types); - let selected: Vec> = positions - .into_iter() - .map(|position| latest[position].clone()) - .collect(); - indexed_set(py, &schema, &selected) -} - -fn indexed_set<'py>( +pub(crate) fn indexed_set<'py>( py: Python<'py>, schema: &str, fingerprints: &[Bound<'py, PyAny>], @@ -112,14 +76,14 @@ fn indexed_set<'py>( new_record(py, RecordId::FingerprintSet, (), Some(&kwargs)) } -fn identity_of(fingerprint: &Bound<'_, PyAny>) -> PyResult<(String, String)> { +pub(crate) fn identity_of(fingerprint: &Bound<'_, PyAny>) -> PyResult<(String, String)> { Ok(( native_field(fingerprint, "node_type")?.extract()?, native_field(fingerprint, "node_name")?.extract()?, )) } -fn borrowed(identities: &[(String, String)]) -> Vec> { +pub(crate) fn borrowed(identities: &[(String, String)]) -> Vec> { identities .iter() .map(|(node_type, node_name)| FingerprintIdentity { @@ -129,45 +93,9 @@ fn borrowed(identities: &[(String, String)]) -> Vec> { .collect() } -pub(crate) fn row_fingerprint<'py>( - fingerprint_class: &Bound<'py, PyAny>, - from_iso: &Bound<'py, PyAny>, - row: &Bound<'py, PyAny>, - qualified_name: &str, -) -> PyResult> { - let py: Python<'py> = row.py(); - let raw_ts: Bound<'py, PyAny> = row.get_item(TIMESTAMP_COLUMN)?; - let ts: Bound<'py, PyAny> = if raw_ts.is_instance_of::() { - raw_ts - } else { - from_iso.call1((raw_ts.str()?,))? - }; - let node_type: Bound<'py, PyString> = row.get_item(0)?.str()?; - let node_name: Bound<'py, PyString> = row.get_item(1)?.str()?; - let raw_target_database: Bound<'py, PyAny> = row.get_item(2)?; - let raw_target_schema: Bound<'py, PyAny> = row.get_item(3)?; - let raw_target_name: Bound<'py, PyAny> = row.get_item(4)?; - let definition_storage: Bound<'py, PyString> = row.get_item(9)?.str()?; - let metadata_json_storage: Bound<'py, PyString> = row.get_item(10)?.str()?; - let definition: String = decoded(&definition_storage, &node_name, qualified_name)?; - let metadata_json: String = decoded(&metadata_json_storage, &node_name, qualified_name)?; - let kwargs: Bound<'py, PyDict> = PyDict::new(py); - kwargs.set_item("node_type", node_type)?; - kwargs.set_item("node_name", node_name)?; - kwargs.set_item("target_database", optional_text(&raw_target_database)?)?; - kwargs.set_item("target_schema", optional_text(&raw_target_schema)?)?; - kwargs.set_item("target_name", optional_text(&raw_target_name)?)?; - kwargs.set_item("run_id", row.get_item(5)?.str()?)?; - kwargs.set_item("definition_hash", row.get_item(6)?.str()?)?; - kwargs.set_item("version_hash", row.get_item(7)?.str()?)?; - kwargs.set_item("schema_fingerprint", row.get_item(8)?.str()?)?; - kwargs.set_item("definition", definition)?; - kwargs.set_item("metadata_json", metadata_json)?; - kwargs.set_item("ts", ts)?; - fingerprint_class.call((), Some(&kwargs)) -} - -fn optional_text<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { +pub(crate) fn optional_text<'py>( + value: &Bound<'py, PyAny>, +) -> PyResult>> { if value.is_none() { return Ok(None); } @@ -175,7 +103,7 @@ fn optional_text<'py>(value: &Bound<'py, PyAny>) -> PyResult, node_name: &Bound<'_, PyString>, qualified_name: &str, @@ -244,8 +172,3 @@ fn decode_error(storage: &Bound<'_, PyString>, failure: StorageDecodeError) -> P } } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(select_latest_fingerprints, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/fingerprints/_helpers/mod.rs b/crates/sqlbuild-python-project/src/fingerprints/_helpers/mod.rs new file mode 100644 index 0000000000..ee85866068 --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod latest_rows; +pub(crate) mod query_hashes; +pub(crate) mod storage_io; +pub(crate) mod storage_sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs b/crates/sqlbuild-python-project/src/fingerprints/_helpers/query_hashes.rs similarity index 56% rename from crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs rename to crates/sqlbuild-python-project/src/fingerprints/_helpers/query_hashes.rs index f9d56acbf5..970087d552 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/_helpers/query_hashes.rs @@ -1,8 +1,7 @@ //! Query and function-definition change fingerprints. -use pyo3::prelude::{Bound, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; +use pyo3::prelude::{PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use sqlbuild_analysis::sql_tokens::main::definition_fingerprint::definition_fingerprint; use sqlbuild_analysis::sql_tokens::main::normalized_query_text::normalized_query_text; @@ -14,7 +13,7 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::types::CompilerDetach; /// `QueryFingerprintError(message)`, or the compiler error a native panic becomes. -fn query_fingerprint_error(py: Python<'_>, message: String) -> PyErr { +pub(crate) fn query_fingerprint_error(py: Python<'_>, message: String) -> PyErr { if is_compiler_panic(&message) { return compiler_error(message); } @@ -24,22 +23,10 @@ fn query_fingerprint_error(py: Python<'_>, message: String) -> PyErr { } } -/// The persisted SHA-256 fingerprint of the query's canonical token stream. -#[pyfunction] -#[pyo3(signature = (*, query_sql, dialect))] -pub(crate) fn compute_query_hash( - py: Python<'_>, - query_sql: &str, - dialect: Option<&str>, -) -> PyResult { - py.compiler_detach(|| compiler_query_fingerprint(query_sql, dialect)) - .map_err(|error| query_fingerprint_error(py, error)) -} - /// A function definition's fingerprint: SQL bodies like queries, other languages exactly. #[pyfunction] #[pyo3(signature = (*, fingerprint_sql, language, dialect))] -fn compute_function_definition_hash( +pub(crate) fn compute_function_definition_hash( py: Python<'_>, fingerprint_sql: &str, language: &str, @@ -51,13 +38,6 @@ fn compute_function_definition_hash( /// Query SQL stripped with its whitespace runs collapsed. #[pyfunction] -fn normalize_query_sql(query_sql: &str) -> String { +pub(crate) fn normalize_query_sql(query_sql: &str) -> String { normalized_query_text(query_sql) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(compute_query_hash, module)?)?; - module.add_function(wrap_pyfunction!(compute_function_definition_hash, module)?)?; - module.add_function(wrap_pyfunction!(normalize_query_sql, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs b/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs rename to crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs index 6bf94189fa..cbf9f38911 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs @@ -2,11 +2,9 @@ use pyo3::IntoPyObject; use pyo3::exceptions::{PyException, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::fingerprint_storage::main::create_table_sql::create_table_sql; use sqlbuild_analysis::fingerprint_storage::main::read_latest_sql::read_latest_sql; use sqlbuild_analysis::fingerprint_storage::main::relevant_read_sql::relevant_read_sql; @@ -14,10 +12,10 @@ use sqlbuild_analysis::fingerprint_storage::models::RelevantRead; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::fingerprints::latest_rows::{ +use crate::fingerprints::_helpers::latest_rows::{ fingerprint_input_error, fingerprint_set_from_rows, }; -use crate::bindings::_helpers::fingerprints::storage_sql::{ +use crate::fingerprints::_helpers::storage_sql::{ fingerprint_insert_batches, fingerprint_insert_sql, }; use sqlbuild_python_base::boundary::models::Request; @@ -276,7 +274,7 @@ fn present<'py>(request: &Request<'_, 'py>, name: &str) -> PyResult, schema: &str, render_qualified_name: &Bound<'_, PyAny>, @@ -293,7 +291,7 @@ fn build_create_table_sql( /// The windowed latest-row read of a target schema's fingerprint table. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name))] -fn build_read_latest_sql( +pub(crate) fn build_read_latest_sql( database: Option<&str>, schema: &str, render_qualified_name: &Bound<'_, PyAny>, @@ -394,7 +392,7 @@ pub(crate) fn read_latest_fingerprints<'py>( /// Append one fingerprint row, retrying transient concurrent-writer conflicts. With `ensure_table` false the table is assumed to exist and is set up only after the insert fails; returns whether the write set up the table. #[pyfunction] #[pyo3(signature = (**request))] -fn write_fingerprint<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult { +pub(crate) fn write_fingerprint<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult { let request = Request::new("write_fingerprint", request)?; let py: Python<'py> = request.py(); let fingerprint: Bound<'py, PyAny> = request.get("fingerprint")?; @@ -448,7 +446,7 @@ fn batch_warning<'py>(batch: &[Bound<'py, PyAny>], error: &PyErr) -> PyResult(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> { +pub(crate) fn write_fingerprints<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> { let request = Request::new("write_fingerprints", request)?; let py: Python<'py> = request.py(); let fingerprints: Vec> = request @@ -489,12 +487,3 @@ fn write_fingerprints<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> } Ok(()) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_create_table_sql, module)?)?; - module.add_function(wrap_pyfunction!(build_read_latest_sql, module)?)?; - module.add_function(wrap_pyfunction!(read_latest_fingerprints, module)?)?; - module.add_function(wrap_pyfunction!(write_fingerprint, module)?)?; - module.add_function(wrap_pyfunction!(write_fingerprints, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs b/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_sql.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs rename to crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_sql.rs index d1fccc23c7..bbc9a407c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs +++ b/crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_sql.rs @@ -7,7 +7,7 @@ use sqlbuild_analysis::fingerprint_storage::main::insert_batch_lengths::insert_b use sqlbuild_analysis::fingerprint_storage::main::insert_sql::insert_sql; use sqlbuild_analysis::fingerprint_storage::models::StoredFingerprint; -use crate::bindings::_helpers::fingerprints::latest_rows::fingerprint_input_error; +use crate::fingerprints::_helpers::latest_rows::fingerprint_input_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// One `INSERT` of a single fingerprint. diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/compute_query_hash.rs b/crates/sqlbuild-python-project/src/fingerprints/main/compute_query_hash.rs new file mode 100644 index 0000000000..3021a285f9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/compute_query_hash.rs @@ -0,0 +1,19 @@ +//! The persisted SHA-256 fingerprint of the query's canonical token stream. + +use crate::fingerprints::_helpers::query_hashes::query_fingerprint_error; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_analysis::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; +use sqlbuild_python_base::types::CompilerDetach; + +/// The persisted SHA-256 fingerprint of the query's canonical token stream. +#[pyfunction] +#[pyo3(signature = (*, query_sql, dialect))] +pub fn compute_query_hash( + py: Python<'_>, + query_sql: &str, + dialect: Option<&str>, +) -> PyResult { + py.compiler_detach(|| compiler_query_fingerprint(query_sql, dialect)) + .map_err(|error| query_fingerprint_error(py, error)) +} diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/mod.rs b/crates/sqlbuild-python-project/src/fingerprints/main/mod.rs new file mode 100644 index 0000000000..ea51d13f10 --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/mod.rs @@ -0,0 +1,7 @@ +pub mod compute_query_hash; +pub mod read_latest_fingerprints; +pub mod register_latest_rows; +pub mod register_query_hashes; +pub mod register_storage_io; +pub mod row_fingerprint; +pub mod select_latest_fingerprints; diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/read_latest_fingerprints.rs b/crates/sqlbuild-python-project/src/fingerprints/main/read_latest_fingerprints.rs new file mode 100644 index 0000000000..d197e13523 --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/read_latest_fingerprints.rs @@ -0,0 +1,13 @@ +//! The latest fingerprint per node identity, read with the adapter-rendered latest-row SQL. + +use crate::fingerprints::_helpers::storage_io; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyDict; + +/// The latest fingerprint per node identity, read with the adapter-rendered latest-row SQL. +#[inline] +pub fn read_latest_fingerprints<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + storage_io::read_latest_fingerprints(request) +} diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/register_latest_rows.rs b/crates/sqlbuild-python-project/src/fingerprints/main/register_latest_rows.rs new file mode 100644 index 0000000000..eebc173054 --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/register_latest_rows.rs @@ -0,0 +1,11 @@ +//! Register the latest rows functions and classes. + +use crate::fingerprints::main::select_latest_fingerprints::select_latest_fingerprints; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the latest rows functions and classes. +pub fn register_latest_rows(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(select_latest_fingerprints, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs b/crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs new file mode 100644 index 0000000000..f7015075e6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs @@ -0,0 +1,15 @@ +//! Register the query hashes functions and classes. + +use crate::fingerprints::_helpers::query_hashes::compute_function_definition_hash; +use crate::fingerprints::_helpers::query_hashes::normalize_query_sql; +use crate::fingerprints::main::compute_query_hash::compute_query_hash; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the query hashes functions and classes. +pub fn register_query_hashes(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compute_query_hash, module)?)?; + module.add_function(wrap_pyfunction!(compute_function_definition_hash, module)?)?; + module.add_function(wrap_pyfunction!(normalize_query_sql, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs b/crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs new file mode 100644 index 0000000000..8ce88cc13e --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs @@ -0,0 +1,19 @@ +//! Register the storage io functions and classes. + +use crate::fingerprints::_helpers::storage_io::build_create_table_sql; +use crate::fingerprints::_helpers::storage_io::build_read_latest_sql; +use crate::fingerprints::_helpers::storage_io::read_latest_fingerprints; +use crate::fingerprints::_helpers::storage_io::write_fingerprint; +use crate::fingerprints::_helpers::storage_io::write_fingerprints; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the storage io functions and classes. +pub fn register_storage_io(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_create_table_sql, module)?)?; + module.add_function(wrap_pyfunction!(build_read_latest_sql, module)?)?; + module.add_function(wrap_pyfunction!(read_latest_fingerprints, module)?)?; + module.add_function(wrap_pyfunction!(write_fingerprint, module)?)?; + module.add_function(wrap_pyfunction!(write_fingerprints, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/row_fingerprint.rs b/crates/sqlbuild-python-project/src/fingerprints/main/row_fingerprint.rs new file mode 100644 index 0000000000..04054c98a2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/row_fingerprint.rs @@ -0,0 +1,46 @@ +//! Row fingerprint. + +use crate::fingerprints::_helpers::latest_rows::TIMESTAMP_COLUMN; +use crate::fingerprints::_helpers::latest_rows::decoded; +use crate::fingerprints::_helpers::latest_rows::optional_text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDateTime, PyDict, PyString}; + +/// Row fingerprint. +pub fn row_fingerprint<'py>( + fingerprint_class: &Bound<'py, PyAny>, + from_iso: &Bound<'py, PyAny>, + row: &Bound<'py, PyAny>, + qualified_name: &str, +) -> PyResult> { + let py: Python<'py> = row.py(); + let raw_ts: Bound<'py, PyAny> = row.get_item(TIMESTAMP_COLUMN)?; + let ts: Bound<'py, PyAny> = if raw_ts.is_instance_of::() { + raw_ts + } else { + from_iso.call1((raw_ts.str()?,))? + }; + let node_type: Bound<'py, PyString> = row.get_item(0)?.str()?; + let node_name: Bound<'py, PyString> = row.get_item(1)?.str()?; + let raw_target_database: Bound<'py, PyAny> = row.get_item(2)?; + let raw_target_schema: Bound<'py, PyAny> = row.get_item(3)?; + let raw_target_name: Bound<'py, PyAny> = row.get_item(4)?; + let definition_storage: Bound<'py, PyString> = row.get_item(9)?.str()?; + let metadata_json_storage: Bound<'py, PyString> = row.get_item(10)?.str()?; + let definition: String = decoded(&definition_storage, &node_name, qualified_name)?; + let metadata_json: String = decoded(&metadata_json_storage, &node_name, qualified_name)?; + let kwargs: Bound<'py, PyDict> = PyDict::new(py); + kwargs.set_item("node_type", node_type)?; + kwargs.set_item("node_name", node_name)?; + kwargs.set_item("target_database", optional_text(&raw_target_database)?)?; + kwargs.set_item("target_schema", optional_text(&raw_target_schema)?)?; + kwargs.set_item("target_name", optional_text(&raw_target_name)?)?; + kwargs.set_item("run_id", row.get_item(5)?.str()?)?; + kwargs.set_item("definition_hash", row.get_item(6)?.str()?)?; + kwargs.set_item("version_hash", row.get_item(7)?.str()?)?; + kwargs.set_item("schema_fingerprint", row.get_item(8)?.str()?)?; + kwargs.set_item("definition", definition)?; + kwargs.set_item("metadata_json", metadata_json)?; + kwargs.set_item("ts", ts)?; + fingerprint_class.call((), Some(&kwargs)) +} diff --git a/crates/sqlbuild-python-project/src/fingerprints/main/select_latest_fingerprints.rs b/crates/sqlbuild-python-project/src/fingerprints/main/select_latest_fingerprints.rs new file mode 100644 index 0000000000..bfc725ca9b --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/main/select_latest_fingerprints.rs @@ -0,0 +1,42 @@ +//! `select_latest_fingerprints`: an unfiltered latest read narrowed as a filtered read would be. + +use crate::fingerprints::_helpers::latest_rows::borrowed; +use crate::fingerprints::_helpers::latest_rows::identity_of; +use crate::fingerprints::_helpers::latest_rows::indexed_set; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_analysis::fingerprint_storage::main::selected_positions::selected_positions; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; + +/// `select_latest_fingerprints`: an unfiltered latest read narrowed as a filtered read would be. +#[pyfunction] +#[pyo3(signature = (*, fingerprint_set, node_names, filtered_node_types = Vec::new()))] +pub fn select_latest_fingerprints<'py>( + py: Python<'py>, + fingerprint_set: &Bound<'py, PyAny>, + node_names: Vec, + filtered_node_types: Vec, +) -> PyResult> { + let schema: String = native_field(fingerprint_set, "schema")?.extract()?; + if node_names.is_empty() && filtered_node_types.is_empty() { + return indexed_set(py, &schema, &[]); + } + let by_identity: Bound<'py, PyAny> = native_field(fingerprint_set, "fingerprints_by_identity")?; + let latest: Vec> = if by_identity.is_truthy()? { + mapping_values(&by_identity)? + .try_iter()? + .collect::>()? + } else { + Vec::new() + }; + let identities: Vec<(String, String)> = + latest.iter().map(identity_of).collect::>()?; + let positions: Vec = + selected_positions(&borrowed(&identities), &node_names, &filtered_node_types); + let selected: Vec> = positions + .into_iter() + .map(|position| latest[position].clone()) + .collect(); + indexed_set(py, &schema, &selected) +} diff --git a/crates/sqlbuild-python-project/src/fingerprints/mod.rs b/crates/sqlbuild-python-project/src/fingerprints/mod.rs new file mode 100644 index 0000000000..94f3b1fa2a --- /dev/null +++ b/crates/sqlbuild-python-project/src/fingerprints/mod.rs @@ -0,0 +1,4 @@ +//! Fingerprint rows: query hashes and their storage. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/mod.rs b/crates/sqlbuild-python-project/src/graph/_helpers/mod.rs new file mode 100644 index 0000000000..5421ba0f18 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod nodes; +pub(crate) mod project; +pub(crate) mod python_nodes; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/nodes/algorithms.rs b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/algorithms.rs new file mode 100644 index 0000000000..6b9118af75 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/algorithms.rs @@ -0,0 +1,89 @@ +//! Closure, edge inversion and path nodes over any Python-keyed edge mapping. + +use crate::graph::main::nodes::closure::closure; +use crate::graph::main::nodes::inverted::inverted; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; +use sqlbuild_core::graph_walk::main::directed_path_nodes::directed_path_nodes; + +use crate::graph::models::PyNodeSet; +use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; + +const BUILTINS_MODULE: &str = "builtins"; + +/// The keys `edges.get(current, ())` lists. +pub(crate) fn neighbours<'py>( + edges: &Bound<'py, PyAny>, + current: &Bound<'py, PyAny>, + none: &Bound<'py, PyTuple>, +) -> PyResult>> { + mapping_get_or(edges, current, none)?.try_iter()?.collect() +} + +/// All graph keys reachable from a starting key. +#[pyfunction] +#[pyo3(signature = (*, start, edges, max_depth = None))] +pub(crate) fn transitive_closure<'py>( + start: Bound<'py, PyAny>, + edges: &Bound<'py, PyAny>, + max_depth: Option, +) -> PyResult> { + let visited: Bound<'py, PySet> = closure(vec![start], edges, (false, max_depth))?; + PyFrozenSet::new(edges.py(), visited.iter()) +} + +/// Reversed directed graph edges, each source list sorted by `sort_key` when given. +#[pyfunction] +#[pyo3(signature = (*, edges, sort_key = None))] +pub(crate) fn invert_edges<'py>( + edges: &Bound<'py, PyAny>, + sort_key: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let py: Python<'py> = edges.py(); + let inverted: Bound<'py, PyDict> = inverted(edges)?; + let sorted: Bound<'py, PyAny> = py.import(BUILTINS_MODULE)?.getattr("sorted")?; + let result: Bound<'py, PyDict> = PyDict::new(py); + for (key, sources) in inverted.iter() { + let sources: Bound<'py, PyAny> = match sort_key { + None => sources, + Some(sort_key) => { + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("key", sort_key)?; + sorted.call((sources,), Some(&keywords))? + } + }; + result.set_item( + key, + PyTuple::new(py, sources.try_iter()?.collect::>>()?)?, + )?; + } + Ok(result) +} + +/// Keys on directed paths between two endpoints, or `None` when `end` is not downstream. +#[pyfunction] +#[pyo3(signature = (*, start, end, downstream))] +pub(crate) fn path_nodes<'py>( + start: Bound<'py, PyAny>, + end: Bound<'py, PyAny>, + downstream: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = downstream.py(); + let upstream: Bound<'py, PyDict> = inverted(downstream)?; + let none: Bound<'py, PyTuple> = PyTuple::empty(py); + let mut reachable: PyNodeSet<'py> = PyNodeSet(PySet::empty(py)?); + let mut path: PyNodeSet<'py> = PyNodeSet(PySet::empty(py)?); + let found: bool = directed_path_nodes( + (start, end), + ( + |current: &Bound<'py, PyAny>| neighbours(downstream, current, &none), + |current: &Bound<'py, PyAny>| neighbours(upstream.as_any(), current, &none), + ), + (&mut reachable, &mut path), + )?; + if !found { + return Ok(None); + } + Ok(Some(PyFrozenSet::new(py, path.0.iter())?)) +} diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/nodes/execution_gates.rs b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/execution_gates.rs new file mode 100644 index 0000000000..11709d459f --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/execution_gates.rs @@ -0,0 +1,18 @@ +//! Ordering edges attached audits and model hooks add, and the keys hooks and SQL refs name. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; + +/// The model config keys whose entries are hooks, in lifecycle order. +pub(crate) const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; + +pub(crate) fn keyword_call<'py>( + callable: &Bound<'py, PyAny>, + pairs: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let keywords: Bound<'py, PyDict> = PyDict::new(callable.py()); + for (name, value) in pairs { + keywords.set_item(name, value)?; + } + callable.call((), Some(&keywords)) +} diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/nodes/mod.rs b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/mod.rs new file mode 100644 index 0000000000..dc46b80f5a --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/nodes/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod algorithms; +pub(crate) mod execution_gates; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/getters.rs b/crates/sqlbuild-python-project/src/graph/_helpers/project/getters.rs similarity index 75% rename from crates/sqlbuild-python/src/bindings/_helpers/graph/getters.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/project/getters.rs index fde060ec0b..fe8f1996e5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/getters.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/project/getters.rs @@ -3,17 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::graph::project_graph::CompiledProjectGraph; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::graph::models::CompiledProjectGraph; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { - class: class_of::, - read: read_compiled_project_graph, -}]; - -fn read_compiled_project_graph<'py>( +pub(crate) fn read_compiled_project_graph<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/project/mod.rs b/crates/sqlbuild-python-project/src/graph/_helpers/project/mod.rs new file mode 100644 index 0000000000..e779cfe25c --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/project/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod getters; +pub(crate) mod project_graph; +pub(crate) mod views; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/project/project_graph.rs b/crates/sqlbuild-python-project/src/graph/_helpers/project/project_graph.rs new file mode 100644 index 0000000000..0bf317f686 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/project/project_graph.rs @@ -0,0 +1,40 @@ +//! The compiled project graph and selector resolution, held natively for Python's graph users. + +use sqlbuild_python_base::positional_record; + +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::errors::SelectorError; + +type Edges = Vec<(ObjectKey, Vec)>; +positional_record! { + extract, into; + /// `(names, upstream, downstream, tags, paths)` in Python's dict order. + pub(crate) struct IndexesRow { + pub(crate) names: Vec<(String, ObjectKey)>, + pub(crate) upstream: Edges, + pub(crate) downstream: Edges, + pub(crate) tags: Vec<(String, Vec)>, + pub(crate) paths: Vec<(ObjectKey, String)>, + } +} +/// `(unique id, neighbour unique ids)` in graph order. +pub(crate) type UniqueIdMap = Vec<(String, Vec)>; +/// `PlannerInputError`'s `(code, message, help)`. +type FailureRow = (&'static str, String, Option); +/// The keys, or the `PlannerInputError` Python raises instead. +pub(crate) type KeysOutcome = (Option>, Option); + +pub(crate) fn dependency_edges(key: &ObjectKey, deps: &[ObjectKey]) -> Vec<(ObjectKey, ObjectKey)> { + deps.iter().map(|dep| (dep.clone(), key.clone())).collect() +} + +pub(crate) fn outcome(result: Result, SelectorError>) -> KeysOutcome { + match result { + Ok(keys) => (Some(keys), None), + Err(error) => (None, Some(failure(error))), + } +} + +fn failure(error: SelectorError) -> FailureRow { + (error.code, error.message, error.help) +} diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/project/views.rs b/crates/sqlbuild-python-project/src/graph/_helpers/project/views.rs new file mode 100644 index 0000000000..ef98ca9639 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/project/views.rs @@ -0,0 +1,117 @@ +//! The native project graph's indexes as the Python dicts graph consumers read, and back. + +use crate::graph::main::nodes::native_key::native_key; +use crate::graph::main::project::edge_dict::edge_dict; +use crate::graph::main::project::graph_from_view_mappings::graph_from_view_mappings; +use crate::graph::models::GraphViewMappings; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; +use sqlbuild_analysis::graph::models::{GraphIndexes, ProjectGraph}; + +use crate::graph::models::KeyCache; +use crate::graph::models::NativeProjectGraph; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_base::records::types::RecordId; + +/// The native graph's pairs for `keys`, in order. +#[pyfunction] +pub(crate) fn native_graph_keys(keys: &Bound<'_, PyAny>) -> PyResult> { + keys.try_iter()?.map(|key| native_key(&key?)).collect() +} + +/// Selector name to key, in project order, without the other graph views. +#[pyfunction] +pub(crate) fn lineage_graph_names<'py>( + py: Python<'py>, + graph: PyRef<'py, NativeProjectGraph>, +) -> PyResult> { + let cache: KeyCache<'py> = KeyCache::new(py)?; + let names: Bound<'py, PyDict> = PyDict::new(py); + for (name, key) in &graph.graph().names { + names.set_item(name, cache.key(key)?)?; + } + Ok(names) +} + +/// The native graph's indexes as `LineageGraphViews` dicts, in project order. +pub(crate) fn graph_views<'py>( + py: Python<'py>, + graph: &ProjectGraph, +) -> PyResult> { + let cache: KeyCache<'py> = KeyCache::new(py)?; + let tags: Bound<'py, PyDict> = PyDict::new(py); + for (tag, keys) in &graph.tags { + tags.set_item(tag, PyFrozenSet::new(py, cache.keys(keys)?)?)?; + } + let paths: Bound<'py, PyDict> = PyDict::new(py); + for (key, folder) in &graph.paths { + paths.set_item(cache.key(key)?, folder)?; + } + let names: Bound<'py, PyDict> = PyDict::new(py); + for (name, key) in &graph.names { + names.set_item(name, cache.key(key)?)?; + } + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("upstream_deps", edge_dict(&cache, &graph.upstream)?)?; + keywords.set_item("downstream_deps", edge_dict(&cache, &graph.downstream)?)?; + keywords.set_item("tag_index", tags)?; + keywords.set_item("path_index", paths)?; + keywords.set_item("all_keys", names)?; + new_record(py, RecordId::LineageGraphViews, (), Some(&keywords)) +} + +/// The graph's lineage edges and selector indexes as dicts, in project order. +#[pyfunction] +pub(crate) fn lineage_graph_views<'py>( + py: Python<'py>, + graph: PyRef<'py, NativeProjectGraph>, +) -> PyResult> { + graph_views(py, graph.graph()) +} + +pub(crate) fn native_edges( + mapping: &Bound<'_, PyAny>, +) -> PyResult)>> { + let mut edges: Vec<(ObjectKey, Vec)> = Vec::new(); + for item in mapping_items(mapping)?.try_iter()? { + let (key, deps): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + edges.push((native_key(&key)?, native_graph_keys(&deps)?)); + } + Ok(edges) +} + +pub(crate) fn graph_from_dicts(indexes: GraphIndexes) -> PyResult { + compiler_guard(|| Ok(NativeProjectGraph::new(graph_from_indexes(indexes)))) +} + +/// A native graph over these indexes, keeping their order. +#[pyfunction] +#[pyo3(signature = (**request))] +pub(crate) fn native_graph_from_views( + request: Option<&Bound<'_, PyDict>>, +) -> PyResult { + let Some(request) = request else { + return Err(pyo3::exceptions::PyTypeError::new_err( + "native_graph_from_views() missing keyword arguments", + )); + }; + let field = |name: &str| -> PyResult> { + request.get_item(name)?.ok_or_else(|| { + pyo3::exceptions::PyTypeError::new_err(format!( + "native_graph_from_views() missing keyword argument '{name}'" + )) + }) + }; + graph_from_view_mappings(GraphViewMappings { + all_keys: &field("all_keys")?, + upstream: &field("upstream")?, + downstream: &field("downstream")?, + tag_index: &field("tag_index")?, + path_index: &field("path_index")?, + }) +} diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/mod.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/mod.rs new file mode 100644 index 0000000000..4388c92d11 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod python_lifecycle; +pub(crate) mod python_node_graph; +pub(crate) mod python_selection; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs new file mode 100644 index 0000000000..b9c88351a2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs @@ -0,0 +1,18 @@ +//! Lifecycle regions of a run's selected Python nodes, classified natively. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::python_plan::models::LifecycleNode; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +const EXTERNAL_CONNECTION_MODE: &str = "external"; + +pub(crate) fn lifecycle_node(node: &Bound<'_, PyAny>) -> PyResult { + let loader: Bound<'_, PyAny> = native_field(node, "loader")?; + let external_loader: bool = !loader.is_none() + && native_field(&loader, "connection_mode")?.extract::()? + == EXTERNAL_CONNECTION_MODE; + Ok(LifecycleNode { + kind: native_field(node, "kind")?.extract()?, + external_loader, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs similarity index 72% rename from crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs index cfebc0428e..3a20a71b6d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs @@ -1,20 +1,16 @@ //! The executable Python-node graph built from discovered project inputs. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::graph::main::python_nodes::python_node_graph::python_node_graph; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; use sqlbuild_analysis::python_selection::models::{InventoryIndexes, InventoryNode}; use sqlbuild_host::entries::models::UserCodeHost; -use sqlbuild_python_base::native_host::main::within_identity_pass::within_identity_pass; -use crate::bindings::_helpers::graph::python_selection::resolve_python_sql_run_selection; -use crate::bindings::_helpers::graph::views::compiled_graph_keys; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::discovery::main::python_files::node_identity::node_identity; @@ -23,7 +19,7 @@ use sqlbuild_python_front::discovery::main::python_files::provider_usages::provi const NAME_KEY: &str = "name"; /// One discovered function collection, its node kind and whether it carries documentation. -const KINDS: [(&str, &str, bool); 4] = [ +pub(crate) const KINDS: [(&str, &str, bool); 4] = [ ("loader_functions", "LOADER", false), ("task_functions", "TASK", true), ("asset_functions", "ASSET", true), @@ -31,11 +27,11 @@ const KINDS: [(&str, &str, bool); 4] = [ ]; /// Python classes and shims every node is built with. -struct Vocabulary<'py> { - node: Bound<'py, PyAny>, - kinds: Bound<'py, PyAny>, - sql_ref: Bound<'py, PyAny>, - providers: Bound<'py, PyDict>, +pub(crate) struct Vocabulary<'py> { + pub(crate) node: Bound<'py, PyAny>, + pub(crate) kinds: Bound<'py, PyAny>, + pub(crate) sql_ref: Bound<'py, PyAny>, + pub(crate) providers: Bound<'py, PyDict>, } fn attribute_value<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { @@ -91,7 +87,7 @@ fn kind_metadata<'py>( } /// One `DiscoveredPythonNode` with its identity and provider usages read by the user shims. -fn python_node<'py>( +pub(crate) fn python_node<'py>( vocabulary: &Vocabulary<'py>, (kind, documented): (&str, bool), discovered: &Bound<'py, PyAny>, @@ -135,7 +131,7 @@ fn python_node<'py>( } /// Each node's upstream Python-node names, resolving decorated dependencies through the shim. -fn upstream_names( +pub(crate) fn upstream_names( py: Python<'_>, nodes: &[Bound<'_, PyAny>], sql_ref: &Bound<'_, PyAny>, @@ -171,7 +167,10 @@ fn upstream_names( .collect() } -fn inventory_node(node: &Bound<'_, PyAny>, upstream: Vec) -> PyResult { +pub(crate) fn inventory_node( + node: &Bound<'_, PyAny>, + upstream: Vec, +) -> PyResult { Ok(InventoryNode { name: native_field(node, "name")?.extract()?, tags: native_field(node, "tags")?.extract()?, @@ -182,42 +181,9 @@ fn inventory_node(node: &Bound<'_, PyAny>, upstream: Vec) -> PyResult( - discovered_inputs: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = discovered_inputs.py(); - let providers: Bound<'py, PyDict> = PyDict::new(py); - for provider in native_field(discovered_inputs, "providers")?.try_iter()? { - let provider: Bound<'py, PyAny> = provider?; - providers.set_item(native_field(&provider, "name")?, provider)?; - } - let vocabulary: Vocabulary<'py> = Vocabulary { - node: record_class(py, RecordId::DiscoveredPythonNode)?, - kinds: host_entry(py, UserCodeHost::PythonNodeKind)?, - sql_ref: host_entry(py, UserCodeHost::SqlResourceRef)?, - providers, - }; - let mut nodes: Vec> = Vec::new(); - within_identity_pass(py, || { - for (collection, kind, documented) in KINDS { - for discovered in native_field(discovered_inputs, collection)?.try_iter()? { - nodes.push(python_node(&vocabulary, (kind, documented), &discovered?)?); - } - } - Ok(()) - })?; - let inventory: Vec = nodes - .iter() - .zip(upstream_names(py, &nodes, &vocabulary.sql_ref)?) - .map(|(node, upstream)| inventory_node(node, upstream)) - .collect::>()?; - graph_record(py, &nodes, graph_indexes(&inventory)) -} - type NameRows = Vec<(String, Vec)>; -fn graph_record<'py>( +pub(crate) fn graph_record<'py>( py: Python<'py>, nodes: &[Bound<'py, PyAny>], indexes: InventoryIndexes, @@ -290,40 +256,12 @@ fn graph_record<'py>( ) } -/// `PythonSqlRunSelection` of run selectors over the project graph and discovered Python nodes. -pub(crate) fn run_selection_from_inputs<'py>( - (select, exclude): (Vec, Vec), - project_graph: &Bound<'py, PyAny>, - discovered_inputs: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_graph.py(); - let python_graph: Bound<'py, PyAny> = python_node_graph(discovered_inputs)?; - let (sql_keys, python_node_names) = - resolve_python_sql_run_selection(project_graph, &python_graph, select, exclude)?; - new_record_with( - py, - RecordId::PythonSqlRunSelection, - &[ - ("sql_keys", compiled_graph_keys(py, sql_keys)?.into_any()), - ( - "python_node_names", - PyFrozenSet::new(py, python_node_names)?.into_any(), - ), - ], - ) -} - /// Build the internal executable Python-node graph from discovered project inputs. #[pyfunction] #[pyo3(signature = (**options))] -fn build_python_node_graph<'py>( +pub(crate) fn build_python_node_graph<'py>( options: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("build_python_node_graph", options)?; python_node_graph(&request.get("discovered_inputs")?) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_python_node_graph, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs similarity index 56% rename from crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs rename to crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs index d578631e41..6d7c157329 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs @@ -1,32 +1,25 @@ //! Python node and unified SQL/Python selector resolution, and the Python node graph indexes. +use crate::graph::main::python_nodes::python_node_names::python_node_names; use sqlbuild_python_base::positional_record; use std::borrow::Cow; -use std::collections::BTreeSet; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyFrozenSet}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::errors::SelectorError; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_analysis::python_selection::main::attach_python_jobs::attach_python_jobs; use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; -use sqlbuild_analysis::python_selection::main::python_run_key::python_run_key; -use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; -use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; -use sqlbuild_analysis::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection as run_selection; use sqlbuild_analysis::python_selection::models::{ InventoryIndexes, InventoryNode, PythonSelectionGraph, PythonSelectionNode, RunSelection, SqlSelectionGraph, }; -use crate::bindings::_helpers::compiled_project::job_catalog::needed_jobs; -use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; +use crate::compiled_project::main::needed_jobs::needed_jobs; +use crate::graph::models::{CompiledProjectGraph, NativeProjectGraph}; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -37,7 +30,7 @@ type NameRows = Vec<(String, Vec)>; positional_record! { extract; /// `(name, tags, POSIX relative path, upstream node names)` of one discovered node. - struct InventoryRow { + pub(crate) struct InventoryRow { name: String, tags: Vec, relative_path: String, @@ -48,7 +41,7 @@ positional_record! { positional_record! { into; /// `(edges, upstream, downstream, tags, paths)` in Python's dict order. - struct IndexesRow { + pub(crate) struct IndexesRow { edges: Vec<(String, String)>, upstream: NameRows, downstream: NameRows, @@ -58,7 +51,7 @@ positional_record! { } /// The `PlannerInputError` Python raises for a selector failure. -fn planner_error(py: Python<'_>, error: SelectorError) -> PyErr { +pub(crate) fn planner_error(py: Python<'_>, error: SelectorError) -> PyErr { let raised = (|| -> PyResult> { let keywords: Bound<'_, PyDict> = PyDict::new(py); keywords.set_item("code", error.code)?; @@ -104,7 +97,7 @@ fn selection_node(node: &Bound<'_, PyAny>) -> PyResult { } /// A `PythonNodeGraph` as the native selection graph. -fn selection_graph(graph: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn selection_graph(graph: &Bound<'_, PyAny>) -> PyResult { Ok(PythonSelectionGraph { nodes: native_field(graph, "nodes")? .try_iter()? @@ -122,7 +115,9 @@ fn selection_graph(graph: &Bound<'_, PyAny>) -> PyResult { } /// `(name, loader)` of every source in the compiled project. -fn project_sources(project: &Bound<'_, PyAny>) -> PyResult)>> { +pub(crate) fn project_sources( + project: &Bound<'_, PyAny>, +) -> PyResult)>> { native_field(project, "sources")? .try_iter()? .map(|source| { @@ -137,118 +132,10 @@ fn project_sources(project: &Bound<'_, PyAny>) -> PyResult)>, - pub(crate) nodes: PythonSelectionGraph, -} - -impl ProjectSelection { - pub(crate) fn new( - project: &Bound<'_, PyAny>, - discovered_inputs: &Bound<'_, PyAny>, - ) -> PyResult { - Ok(Self { - sources: project_sources(project)?, - nodes: selection_graph(&python_node_graph(discovered_inputs)?)?, - }) - } - - /// The `kind:name` keys `sqb build --select` runs for these selectors. - pub(crate) fn run_keys( - &self, - graph: &ProjectGraph, - select: &[String], - exclude: &[String], - ) -> Result, String> { - self.keys(run_selection(self.sql(graph), &self.nodes, select, exclude)) - } - - /// The `kind:name` keys these selectors name, Python checks included. - pub(crate) fn node_keys( - &self, - graph: &ProjectGraph, - select: &[String], - exclude: &[String], - ) -> Result, String> { - self.keys(resolve_python_sql_nodes( - self.sql(graph), - &self.nodes, - select, - exclude, - )) - } - - /// The published key of a Python node, such as `task:export_orders`. - pub(crate) fn node_key(&self, name: &str) -> Option { - self.nodes - .nodes - .iter() - .rev() - .find(|node| node.name == name) - .map(|node| format!("{}:{name}", node.kind)) - } - - /// The key a selected Python node's run reports under; a loader reports as the source it writes. - fn run_key(&self, name: &str) -> Option { - let node = self - .nodes - .nodes - .iter() - .rev() - .find(|node| node.name == name)?; - Some(python_run_key(&self.sources, (&node.kind, name))) - } - - /// Whether a loader writes no declared source and so runs only as part of another loader's chain. - pub(crate) fn intermediate_loader(&self, name: &str) -> bool { - !self - .sources - .iter() - .any(|(_, loader)| loader.as_deref() == Some(name)) - } - - fn sql<'graph>(&'graph self, graph: &'graph ProjectGraph) -> SqlSelectionGraph<'graph> { - SqlSelectionGraph { - graph, - sources: &self.sources, - } - } - - fn keys( - &self, - selection: Result, - ) -> Result, String> { - let selection: RunSelection = selection.map_err(|error| error.message)?; - let mut keys: BTreeSet = selection - .sql_keys - .into_iter() - .map(|(kind, name)| format!("{kind}:{name}")) - .collect(); - for name in &selection.python_node_names { - keys.extend(self.run_key(name)); - } - Ok(keys) - } -} - -/// The Python node names `select` minus `exclude` resolve to; every node when unselected. -pub(crate) fn python_node_names<'py>( - graph: &Bound<'py, PyAny>, - (select, exclude): (Vec, Vec), -) -> PyResult> { - let py: Python<'py> = graph.py(); - let native: PythonSelectionGraph = selection_graph(graph)?; - let names: Vec = - compiler_guard(|| Ok(resolve_python_node_selectors(&native, &select, &exclude)))? - .map_err(|error| planner_error(py, error))?; - PyFrozenSet::new(py, names) -} - /// The Python node names `select` minus `exclude` resolve to; every node when unselected. #[pyfunction] #[pyo3(signature = (*, graph, select, exclude))] -fn resolve_python_node_selection<'py>( +pub(crate) fn resolve_python_node_selection<'py>( graph: &Bound<'py, PyAny>, select: Vec, exclude: Vec, @@ -256,21 +143,6 @@ fn resolve_python_node_selection<'py>( python_node_names(graph, (select, exclude)) } -/// The Python node names `sqb check` exclude tokens name, skipping tokens that match nothing. -#[pyfunction] -#[pyo3(signature = (*, graph, exclude))] -pub(crate) fn resolve_python_check_excludes<'py>( - graph: &Bound<'py, PyAny>, - exclude: Vec, -) -> PyResult> { - let py: Python<'py> = graph.py(); - let native: PythonSelectionGraph = selection_graph(graph)?; - let names: Vec = - compiler_guard(|| Ok(resolve_python_node_excludes(&native, &exclude)))? - .map_err(|error| planner_error(py, error))?; - PyFrozenSet::new(py, names) -} - /// `(sql keys, python node names)` run selectors resolve to across SQL resources and Python nodes. #[pyfunction] #[pyo3(signature = (*, project_graph, python_graph, select, exclude))] @@ -316,7 +188,7 @@ pub(crate) fn resolve_python_sql_run_selection( /// The dependency edges, adjacency, tag and path indexes of discovered Python nodes. #[pyfunction] -fn python_node_graph_indexes(nodes: Vec) -> PyResult { +pub(crate) fn python_node_graph_indexes(nodes: Vec) -> PyResult { compiler_guard(|| { let nodes: Vec = nodes .into_iter() @@ -350,10 +222,3 @@ fn python_node_graph_indexes(nodes: Vec) -> PyResult { }) }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_python_node_selection, module)?)?; - module.add_function(wrap_pyfunction!(resolve_python_check_excludes, module)?)?; - module.add_function(wrap_pyfunction!(resolve_python_sql_run_selection, module)?)?; - module.add_function(wrap_pyfunction!(python_node_graph_indexes, module)?) -} diff --git a/crates/sqlbuild-python-project/src/graph/constants.rs b/crates/sqlbuild-python-project/src/graph/constants.rs new file mode 100644 index 0000000000..574ce12638 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/constants.rs @@ -0,0 +1,12 @@ +//! Public constants of graph. + +use crate::graph::_helpers::project::getters::read_compiled_project_graph; +use crate::graph::models::CompiledProjectGraph; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[GetterEntry { + class: class_of::, + read: read_compiled_project_graph, +}]; diff --git a/crates/sqlbuild-python-project/src/graph/main/mod.rs b/crates/sqlbuild-python-project/src/graph/main/mod.rs new file mode 100644 index 0000000000..5db9403881 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/mod.rs @@ -0,0 +1,3 @@ +pub mod nodes; +pub mod project; +pub mod python_nodes; diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/closure.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/closure.rs new file mode 100644 index 0000000000..8c353124fd --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/closure.rs @@ -0,0 +1,30 @@ +//! Keys within `max_depth` edges of any start, plus the starts when `include_starts`. + +use crate::graph::_helpers::nodes::algorithms::neighbours; +use crate::graph::models::PyNodeSet; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::{PySet, PySetMethods, PyTuple}; +use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; + +/// Keys within `max_depth` edges of any start, plus the starts when `include_starts`. +pub fn closure<'py>( + starts: Vec>, + edges: &Bound<'py, PyAny>, + (include_starts, max_depth): (bool, Option), +) -> PyResult> { + let py: Python<'py> = edges.py(); + let mut visited: PyNodeSet<'py> = PyNodeSet(PySet::empty(py)?); + if include_starts { + for start in &starts { + visited.0.add(start)?; + } + } + let none: Bound<'py, PyTuple> = PyTuple::empty(py); + breadth_first_reach( + starts, + max_depth.map(|maximum| u64::try_from(maximum).unwrap_or(0)), + &mut visited, + |current| neighbours(edges, current, &none), + )?; + Ok(visited.0) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/inverted.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/inverted.rs new file mode 100644 index 0000000000..831736a48c --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/inverted.rs @@ -0,0 +1,30 @@ +//! Reversed edges: every source key first, then each new target in first-seen order. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods}; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; + +/// Reversed edges: every source key first, then each new target in first-seen order. +pub fn inverted<'py>(edges: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = edges.py(); + let inverted: Bound<'py, PyDict> = PyDict::new(py); + for key in edges.try_iter()? { + inverted.set_item(key?, PyList::empty(py))?; + } + for item in mapping_items(edges)?.try_iter()? { + let (source, targets): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + for target in targets.try_iter()? { + let target: Bound<'py, PyAny> = target?; + let sources: Bound<'py, PyList> = match inverted.get_item(&target)? { + Some(sources) => sources.downcast_into()?, + None => { + let sources: Bound<'py, PyList> = PyList::empty(py); + inverted.set_item(&target, &sources)?; + sources + } + }; + sources.append(&source)?; + } + } + Ok(inverted) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/mod.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/mod.rs new file mode 100644 index 0000000000..c79e1b9d5e --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/mod.rs @@ -0,0 +1,8 @@ +pub mod closure; +pub mod inverted; +pub mod model_python_hook_names; +pub mod native_key; +pub mod register_algorithms; +pub mod register_execution_gates; +pub mod sql_ref_key; +pub mod transitive_closure_many; diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/model_python_hook_names.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/model_python_hook_names.rs new file mode 100644 index 0000000000..0b665c5f0e --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/model_python_hook_names.rs @@ -0,0 +1,42 @@ +//! The Python hooks attached to one model, in lifecycle order without repeats. + +use crate::graph::_helpers::nodes::execution_gates::MODEL_HOOK_KEYS; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyList, PyTuple}; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_base::records::types::RecordId; + +/// The Python hooks attached to one model, in lifecycle order without repeats. +#[pyfunction] +#[pyo3(signature = (*, model))] +pub fn model_python_hook_names<'py>( + py: Python<'py>, + model: &Bound<'py, PyAny>, +) -> PyResult> { + let entry_type: Bound<'_, PyAny> = record_class(py, RecordId::PythonHookEntry)?; + let values: Bound<'_, PyAny> = native_field(model, "config")?.getattr("values")?; + let mut names: Vec = Vec::new(); + for hook_key in MODEL_HOOK_KEYS { + let entries: Bound<'_, PyAny> = mapping_get(&values, hook_key)?; + if entries.is_instance(&entry_type)? { + names.push(native_field(&entries, "name")?.extract()?); + } else if entries.is_instance_of::() || entries.is_instance_of::() { + for entry in entries.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + if entry.is_instance(&entry_type)? { + names.push(native_field(&entry, "name")?.extract()?); + } + } + } + } + let mut unique: Vec = Vec::with_capacity(names.len()); + for name in names { + if !unique.contains(&name) { + unique.push(name); + } + } + PyTuple::new(py, unique) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs new file mode 100644 index 0000000000..d7d6e4ea3f --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs @@ -0,0 +1,12 @@ +//! The native `(resource type, name)` pair of a `CompiledObjectKey`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The native `(resource type, name)` pair of a `CompiledObjectKey`. +pub fn native_key(key: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + Ok(( + native_field(key, "resource_type")?.str()?.extract()?, + native_field(key, "name")?.extract()?, + )) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs new file mode 100644 index 0000000000..dca2b53cc1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs @@ -0,0 +1,17 @@ +//! Register the algorithms functions and classes. + +use crate::graph::_helpers::nodes::algorithms::invert_edges; +use crate::graph::_helpers::nodes::algorithms::path_nodes; +use crate::graph::_helpers::nodes::algorithms::transitive_closure; +use crate::graph::main::nodes::transitive_closure_many::transitive_closure_many; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the algorithms functions and classes. +pub fn register_algorithms(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(transitive_closure, module)?)?; + module.add_function(wrap_pyfunction!(transitive_closure_many, module)?)?; + module.add_function(wrap_pyfunction!(invert_edges, module)?)?; + module.add_function(wrap_pyfunction!(path_nodes, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/register_execution_gates.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/register_execution_gates.rs new file mode 100644 index 0000000000..424f5bdc42 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/register_execution_gates.rs @@ -0,0 +1,13 @@ +//! Register the execution gates functions and classes. + +use crate::graph::main::nodes::model_python_hook_names::model_python_hook_names; +use crate::graph::main::nodes::sql_ref_key::sql_ref_key; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the execution gates functions and classes. +pub fn register_execution_gates(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(model_python_hook_names, module)?)?; + module.add_function(wrap_pyfunction!(sql_ref_key, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/sql_ref_key.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/sql_ref_key.rs new file mode 100644 index 0000000000..28bb786901 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/sql_ref_key.rs @@ -0,0 +1,29 @@ +//! The compiled graph key a typed model, source or seed reference names. + +use crate::graph::_helpers::nodes::execution_gates::keyword_call; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +/// The compiled graph key a typed model, source or seed reference names. +#[pyfunction] +pub fn sql_ref_key<'py>( + py: Python<'py>, + reference: &Bound<'py, PyAny>, +) -> PyResult> { + let resource_type: Bound<'py, PyAny> = enum_from_value( + py, + RecordId::CompiledResourceType, + native_field(reference, "kind")?.getattr("value")?, + )?; + keyword_call( + &record_class(py, RecordId::CompiledObjectKey)?, + &[ + ("resource_type", resource_type), + ("name", native_field(reference, "name")?), + ], + ) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/nodes/transitive_closure_many.rs b/crates/sqlbuild-python-project/src/graph/main/nodes/transitive_closure_many.rs new file mode 100644 index 0000000000..6d35aa5f5b --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/nodes/transitive_closure_many.rs @@ -0,0 +1,20 @@ +//! Keys within `max_depth` edges of any start, plus the starts when requested. + +use crate::graph::main::nodes::closure::closure; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyFrozenSet, PySet, PySetMethods}; + +/// Keys within `max_depth` edges of any start, plus the starts when requested. +#[pyfunction] +#[pyo3(signature = (*, starts, edges, include_starts, max_depth = None))] +pub fn transitive_closure_many<'py>( + starts: &Bound<'py, PyAny>, + edges: &Bound<'py, PyAny>, + include_starts: bool, + max_depth: Option, +) -> PyResult> { + let starts: Vec> = starts.try_iter()?.collect::>()?; + let visited: Bound<'py, PySet> = closure(starts, edges, (include_starts, max_depth))?; + PyFrozenSet::new(edges.py(), visited.iter()) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/project/compiled_graph_keys.rs b/crates/sqlbuild-python-project/src/graph/main/project/compiled_graph_keys.rs new file mode 100644 index 0000000000..1f9556250d --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/project/compiled_graph_keys.rs @@ -0,0 +1,17 @@ +//! `CompiledObjectKey`s for native graph key pairs. + +use crate::graph::models::KeyCache; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyFrozenSet; +use sqlbuild_analysis::assembly::project::types::ObjectKey; + +/// `CompiledObjectKey`s for native graph key pairs. +#[pyfunction] +pub fn compiled_graph_keys<'py>( + py: Python<'py>, + keys: Vec, +) -> PyResult> { + let cache: KeyCache<'py> = KeyCache::new(py)?; + PyFrozenSet::new(py, cache.keys(&keys)?) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/project/edge_dict.rs b/crates/sqlbuild-python-project/src/graph/main/project/edge_dict.rs new file mode 100644 index 0000000000..3bc1defbff --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/project/edge_dict.rs @@ -0,0 +1,19 @@ +//! Edge dict. + +use crate::graph::models::KeyCache; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; + +/// Edge dict. +pub fn edge_dict<'py>( + cache: &KeyCache<'py>, + edges: &[(ObjectKey, Vec)], +) -> PyResult> { + let py: Python<'py> = cache.py(); + let dict: Bound<'py, PyDict> = PyDict::new(py); + for (key, deps) in edges { + dict.set_item(cache.key(key)?, PyTuple::new(py, cache.keys(deps)?)?)?; + } + Ok(dict) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/project/graph_from_view_mappings.rs b/crates/sqlbuild-python-project/src/graph/main/project/graph_from_view_mappings.rs new file mode 100644 index 0000000000..6ce8b63b45 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/project/graph_from_view_mappings.rs @@ -0,0 +1,40 @@ +//! A native graph over Python's index mappings, keeping their order. + +use crate::graph::_helpers::project::views::graph_from_dicts; +use crate::graph::_helpers::project::views::native_edges; +use crate::graph::_helpers::project::views::native_graph_keys; +use crate::graph::main::nodes::native_key::native_key; +use crate::graph::models::GraphViewMappings; +use crate::graph::models::NativeProjectGraph; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::models::GraphIndexes; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; + +/// A native graph over Python's index mappings, keeping their order. +pub fn graph_from_view_mappings( + mappings: GraphViewMappings<'_, '_>, +) -> PyResult { + let mut names: Vec<(String, ObjectKey)> = Vec::new(); + for item in mapping_items(mappings.all_keys)?.try_iter()? { + let (name, key): (String, Bound<'_, PyAny>) = item?.extract()?; + names.push((name, native_key(&key)?)); + } + let mut tags: Vec<(String, Vec)> = Vec::new(); + for item in mapping_items(mappings.tag_index)?.try_iter()? { + let (tag, keys): (String, Bound<'_, PyAny>) = item?.extract()?; + tags.push((tag, native_graph_keys(&keys)?)); + } + let mut paths: Vec<(ObjectKey, String)> = Vec::new(); + for item in mapping_items(mappings.path_index)?.try_iter()? { + let (key, folder): (Bound<'_, PyAny>, String) = item?.extract()?; + paths.push((native_key(&key)?, folder)); + } + graph_from_dicts(GraphIndexes { + names, + upstream: native_edges(mappings.upstream)?, + downstream: native_edges(mappings.downstream)?, + tags, + paths, + }) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/project/mod.rs b/crates/sqlbuild-python-project/src/graph/main/project/mod.rs new file mode 100644 index 0000000000..d1aee43338 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/project/mod.rs @@ -0,0 +1,8 @@ +pub mod compiled_graph_keys; +pub mod edge_dict; +pub mod graph_from_view_mappings; +pub mod native_graph_from_edges; +pub mod project_graph; +pub mod project_lineage_views; +pub mod register_project_graph; +pub mod register_views; diff --git a/crates/sqlbuild-python-project/src/graph/main/project/native_graph_from_edges.rs b/crates/sqlbuild-python-project/src/graph/main/project/native_graph_from_edges.rs new file mode 100644 index 0000000000..40fa31ee1f --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/project/native_graph_from_edges.rs @@ -0,0 +1,24 @@ +//! A native graph holding these edges in their order, with no selector indexes. + +use crate::graph::_helpers::project::views::graph_from_dicts; +use crate::graph::_helpers::project::views::native_edges; +use crate::graph::models::NativeProjectGraph; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_analysis::graph::models::GraphIndexes; + +/// A native graph holding these edges in their order, with no selector indexes. +#[pyfunction] +#[pyo3(signature = (*, upstream, downstream))] +pub fn native_graph_from_edges( + upstream: &Bound<'_, PyAny>, + downstream: &Bound<'_, PyAny>, +) -> PyResult { + graph_from_dicts(GraphIndexes { + names: Vec::new(), + upstream: native_edges(upstream)?, + downstream: native_edges(downstream)?, + tags: Vec::new(), + paths: Vec::new(), + }) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/project/project_graph.rs b/crates/sqlbuild-python-project/src/graph/main/project/project_graph.rs new file mode 100644 index 0000000000..eded391051 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/project/project_graph.rs @@ -0,0 +1,31 @@ +//! The project graph indexes of these resources, in project order. + +use crate::graph::models::NativeProjectGraph; +use crate::graph::models::ResourceRow; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; +use sqlbuild_analysis::graph::models::GraphResource; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; + +/// The project graph indexes of these resources, in project order. +pub fn project_graph(resources: Vec) -> PyResult { + compiler_guard(|| { + let resources: Vec = resources + .into_iter() + .map( + |ResourceRow { + key, + deps, + tags, + folder, + }| GraphResource { + key, + deps, + tags, + folder, + }, + ) + .collect(); + Ok(NativeProjectGraph::new(build_project_graph(&resources))) + }) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/project/project_lineage_views.rs b/crates/sqlbuild-python-project/src/graph/main/project/project_lineage_views.rs new file mode 100644 index 0000000000..6064feaa28 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/project/project_lineage_views.rs @@ -0,0 +1,18 @@ +//! The project's lineage edges and selector indexes, from its native graph. + +use crate::graph::_helpers::project::views::graph_views; +use crate::graph::models::NativeProjectGraph; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The project's lineage edges and selector indexes, from its native graph. +#[pyfunction] +pub fn project_lineage_views<'py>( + py: Python<'py>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + let graph: PyRef<'py, NativeProjectGraph> = + native_field(project, "lineage_graph")?.extract()?; + graph_views(py, graph.graph()) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/project/register_project_graph.rs b/crates/sqlbuild-python-project/src/graph/main/project/register_project_graph.rs new file mode 100644 index 0000000000..6bdd076d1c --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/project/register_project_graph.rs @@ -0,0 +1,11 @@ +//! Register the project graph functions and classes. + +use crate::graph::models::CompiledProjectGraph; +use crate::graph::models::NativeProjectGraph; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the project graph functions and classes. +pub fn register_project_graph(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::() +} diff --git a/crates/sqlbuild-python-project/src/graph/main/project/register_views.rs b/crates/sqlbuild-python-project/src/graph/main/project/register_views.rs new file mode 100644 index 0000000000..c89a849f29 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/project/register_views.rs @@ -0,0 +1,23 @@ +//! Register the views functions and classes. + +use crate::graph::_helpers::project::views::lineage_graph_names; +use crate::graph::_helpers::project::views::lineage_graph_views; +use crate::graph::_helpers::project::views::native_graph_from_views; +use crate::graph::_helpers::project::views::native_graph_keys; +use crate::graph::main::project::compiled_graph_keys::compiled_graph_keys; +use crate::graph::main::project::native_graph_from_edges::native_graph_from_edges; +use crate::graph::main::project::project_lineage_views::project_lineage_views; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the views functions and classes. +pub fn register_views(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(native_graph_keys, module)?)?; + module.add_function(wrap_pyfunction!(compiled_graph_keys, module)?)?; + module.add_function(wrap_pyfunction!(lineage_graph_names, module)?)?; + module.add_function(wrap_pyfunction!(lineage_graph_views, module)?)?; + module.add_function(wrap_pyfunction!(project_lineage_views, module)?)?; + module.add_function(wrap_pyfunction!(native_graph_from_views, module)?)?; + module.add_function(wrap_pyfunction!(native_graph_from_edges, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs similarity index 72% rename from crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs rename to crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs index 03bc0a19c2..3c85e9cb2d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs @@ -1,22 +1,20 @@ -//! Lifecycle regions of a run's selected Python nodes, classified natively. +//! `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. -use std::collections::{BTreeSet, HashMap}; - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::graph::_helpers::python_nodes::python_lifecycle::lifecycle_node; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyFrozenSet}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::python_plan::main::run_lifecycle::run_lifecycle; use sqlbuild_analysis::python_plan::models::{LifecycleNode, RunLifecycle}; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; - -const EXTERNAL_CONNECTION_MODE: &str = "external"; +use std::collections::{BTreeSet, HashMap}; /// `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. #[pyfunction] #[pyo3(signature = (*, selection, python_graph))] -pub(crate) fn build_python_sql_run_lifecycle<'py>( +pub fn build_python_sql_run_lifecycle<'py>( selection: &Bound<'py, PyAny>, python_graph: &Bound<'py, PyAny>, ) -> PyResult> { @@ -66,18 +64,3 @@ pub(crate) fn build_python_sql_run_lifecycle<'py>( Some(&arguments), ) } - -fn lifecycle_node(node: &Bound<'_, PyAny>) -> PyResult { - let loader: Bound<'_, PyAny> = native_field(node, "loader")?; - let external_loader: bool = !loader.is_none() - && native_field(&loader, "connection_mode")?.extract::()? - == EXTERNAL_CONNECTION_MODE; - Ok(LifecycleNode { - kind: native_field(node, "kind")?.extract()?, - external_loader, - }) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_python_sql_run_lifecycle, module)?) -} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs new file mode 100644 index 0000000000..07eb988543 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs @@ -0,0 +1,8 @@ +pub mod build_python_sql_run_lifecycle; +pub mod python_node_graph; +pub mod python_node_names; +pub mod register_python_lifecycle; +pub mod register_python_node_graph; +pub mod register_python_selection; +pub mod resolve_python_check_excludes; +pub mod run_selection_from_inputs; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs new file mode 100644 index 0000000000..53c6bd7bfa --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_graph.rs @@ -0,0 +1,51 @@ +//! The internal executable Python-node graph of discovered project inputs. + +use crate::graph::_helpers::python_nodes::python_node_graph::KINDS; +use crate::graph::_helpers::python_nodes::python_node_graph::Vocabulary; +use crate::graph::_helpers::python_nodes::python_node_graph::graph_record; +use crate::graph::_helpers::python_nodes::python_node_graph::inventory_node; +use crate::graph::_helpers::python_nodes::python_node_graph::python_node; +use crate::graph::_helpers::python_nodes::python_node_graph::upstream_names; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; +use sqlbuild_analysis::python_selection::models::InventoryNode; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::main::within_identity_pass::within_identity_pass; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +/// The internal executable Python-node graph of discovered project inputs. +pub fn python_node_graph<'py>( + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered_inputs.py(); + let providers: Bound<'py, PyDict> = PyDict::new(py); + for provider in native_field(discovered_inputs, "providers")?.try_iter()? { + let provider: Bound<'py, PyAny> = provider?; + providers.set_item(native_field(&provider, "name")?, provider)?; + } + let vocabulary: Vocabulary<'py> = Vocabulary { + node: record_class(py, RecordId::DiscoveredPythonNode)?, + kinds: host_entry(py, UserCodeHost::PythonNodeKind)?, + sql_ref: host_entry(py, UserCodeHost::SqlResourceRef)?, + providers, + }; + let mut nodes: Vec> = Vec::new(); + within_identity_pass(py, || { + for (collection, kind, documented) in KINDS { + for discovered in native_field(discovered_inputs, collection)?.try_iter()? { + nodes.push(python_node(&vocabulary, (kind, documented), &discovered?)?); + } + } + Ok(()) + })?; + let inventory: Vec = nodes + .iter() + .zip(upstream_names(py, &nodes, &vocabulary.sql_ref)?) + .map(|(node, upstream)| inventory_node(node, upstream)) + .collect::>()?; + graph_record(py, &nodes, graph_indexes(&inventory)) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_names.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_names.rs new file mode 100644 index 0000000000..d729b6c6dd --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_names.rs @@ -0,0 +1,22 @@ +//! The Python node names `select` minus `exclude` resolve to; every node when unselected. + +use crate::graph::_helpers::python_nodes::python_selection::planner_error; +use crate::graph::_helpers::python_nodes::python_selection::selection_graph; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyFrozenSet; +use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; +use sqlbuild_analysis::python_selection::models::PythonSelectionGraph; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; + +/// The Python node names `select` minus `exclude` resolve to; every node when unselected. +pub fn python_node_names<'py>( + graph: &Bound<'py, PyAny>, + (select, exclude): (Vec, Vec), +) -> PyResult> { + let py: Python<'py> = graph.py(); + let native: PythonSelectionGraph = selection_graph(graph)?; + let names: Vec = + compiler_guard(|| Ok(resolve_python_node_selectors(&native, &select, &exclude)))? + .map_err(|error| planner_error(py, error))?; + PyFrozenSet::new(py, names) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_lifecycle.rs new file mode 100644 index 0000000000..2d5251fcc9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_lifecycle.rs @@ -0,0 +1,10 @@ +//! Register the python lifecycle functions and classes. + +use crate::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the python lifecycle functions and classes. +pub fn register_python_lifecycle(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_python_sql_run_lifecycle, module)?) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_node_graph.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_node_graph.rs new file mode 100644 index 0000000000..1827619dab --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_node_graph.rs @@ -0,0 +1,11 @@ +//! Register the python node graph functions and classes. + +use crate::graph::_helpers::python_nodes::python_node_graph::build_python_node_graph; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the python node graph functions and classes. +pub fn register_python_node_graph(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_python_node_graph, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs new file mode 100644 index 0000000000..37fc6b5f03 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs @@ -0,0 +1,16 @@ +//! Register the python selection functions and classes. + +use crate::graph::_helpers::python_nodes::python_selection::python_node_graph_indexes; +use crate::graph::_helpers::python_nodes::python_selection::resolve_python_node_selection; +use crate::graph::_helpers::python_nodes::python_selection::resolve_python_sql_run_selection; +use crate::graph::main::python_nodes::resolve_python_check_excludes::resolve_python_check_excludes; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the python selection functions and classes. +pub fn register_python_selection(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_python_node_selection, module)?)?; + module.add_function(wrap_pyfunction!(resolve_python_check_excludes, module)?)?; + module.add_function(wrap_pyfunction!(resolve_python_sql_run_selection, module)?)?; + module.add_function(wrap_pyfunction!(python_node_graph_indexes, module)?) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_python_check_excludes.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_python_check_excludes.rs new file mode 100644 index 0000000000..6b9ba1bc31 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_python_check_excludes.rs @@ -0,0 +1,25 @@ +//! The Python node names `sqb check` exclude tokens name, skipping tokens that match nothing. + +use crate::graph::_helpers::python_nodes::python_selection::planner_error; +use crate::graph::_helpers::python_nodes::python_selection::selection_graph; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyFrozenSet; +use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; +use sqlbuild_analysis::python_selection::models::PythonSelectionGraph; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; + +/// The Python node names `sqb check` exclude tokens name, skipping tokens that match nothing. +#[pyfunction] +#[pyo3(signature = (*, graph, exclude))] +pub fn resolve_python_check_excludes<'py>( + graph: &Bound<'py, PyAny>, + exclude: Vec, +) -> PyResult> { + let py: Python<'py> = graph.py(); + let native: PythonSelectionGraph = selection_graph(graph)?; + let names: Vec = + compiler_guard(|| Ok(resolve_python_node_excludes(&native, &exclude)))? + .map_err(|error| planner_error(py, error))?; + PyFrozenSet::new(py, names) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/run_selection_from_inputs.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/run_selection_from_inputs.rs new file mode 100644 index 0000000000..6006812be3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/run_selection_from_inputs.rs @@ -0,0 +1,32 @@ +//! `PythonSqlRunSelection` of run selectors over the project graph and discovered Python nodes. + +use crate::graph::_helpers::python_nodes::python_selection::resolve_python_sql_run_selection; +use crate::graph::main::project::compiled_graph_keys::compiled_graph_keys; +use crate::graph::main::python_nodes::python_node_graph::python_node_graph; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyFrozenSet; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; + +/// `PythonSqlRunSelection` of run selectors over the project graph and discovered Python nodes. +pub fn run_selection_from_inputs<'py>( + (select, exclude): (Vec, Vec), + project_graph: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_graph.py(); + let python_graph: Bound<'py, PyAny> = python_node_graph(discovered_inputs)?; + let (sql_keys, python_node_names) = + resolve_python_sql_run_selection(project_graph, &python_graph, select, exclude)?; + new_record_with( + py, + RecordId::PythonSqlRunSelection, + &[ + ("sql_keys", compiled_graph_keys(py, sql_keys)?.into_any()), + ( + "python_node_names", + PyFrozenSet::new(py, python_node_names)?.into_any(), + ), + ], + ) +} diff --git a/crates/sqlbuild-python-project/src/graph/mod.rs b/crates/sqlbuild-python-project/src/graph/mod.rs new file mode 100644 index 0000000000..10b27983f7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/mod.rs @@ -0,0 +1,6 @@ +//! The project graph: keys, node sets, algorithms, execution gates and Python nodes. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-project/src/graph/models.rs b/crates/sqlbuild-python-project/src/graph/models.rs new file mode 100644 index 0000000000..c55cc47781 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/models.rs @@ -0,0 +1,496 @@ +//! Public records of graph. + +use crate::graph::_helpers::project::project_graph::IndexesRow; +use crate::graph::_helpers::project::project_graph::KeysOutcome; +use crate::graph::_helpers::project::project_graph::UniqueIdMap; +use crate::graph::_helpers::project::project_graph::dependency_edges; +use crate::graph::_helpers::project::project_graph::outcome; +use crate::graph::_helpers::project::views::lineage_graph_views; +use crate::graph::_helpers::python_nodes::python_selection::project_sources; +use crate::graph::_helpers::python_nodes::python_selection::selection_graph; +use crate::graph::main::project::project_graph::project_graph; +use crate::graph::main::python_nodes::python_node_graph::python_node_graph; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::sync::PyOnceLock; +use pyo3::types::PySet; +use pyo3::types::PySetMethods; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::errors::SelectorError; +use sqlbuild_analysis::graph::main::build_resources::build_resources; +use sqlbuild_analysis::graph::main::edge_keys::edge_keys; +use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; +use sqlbuild_analysis::graph::main::match_selector::match_selector; +use sqlbuild_analysis::graph::main::model_layer_count::model_layer_count; +use sqlbuild_analysis::graph::main::resolve_selector_tokens::resolve_selector_tokens; +use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; +use sqlbuild_analysis::graph::main::unique_id_edges::unique_id_edges; +use sqlbuild_analysis::graph::models::BuildResources; +use sqlbuild_analysis::graph::models::EdgeDirection; +use sqlbuild_analysis::graph::models::GraphIndexes; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_analysis::python_selection::main::python_run_key::python_run_key; +use sqlbuild_analysis::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; +use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection as run_selection; +use sqlbuild_analysis::python_selection::models::PythonSelectionGraph; +use sqlbuild_analysis::python_selection::models::RunSelection; +use sqlbuild_analysis::python_selection::models::SqlSelectionGraph; +use sqlbuild_core::graph_walk::types::NodeSet; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::construct::record_class::record_class; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use std::cell::RefCell; +use std::collections::BTreeSet; +use std::collections::HashMap; + +/// One `CompiledObjectKey` per native pair, so shared keys convert once. +#[derive(Debug)] +pub struct KeyCache<'py> { + pub(crate) key_type: Bound<'py, PyAny>, + pub(crate) resource_type: Bound<'py, PyAny>, + pub(crate) keys: RefCell>>, +} + +impl<'py> KeyCache<'py> { + pub fn new(py: Python<'py>) -> PyResult { + Ok(Self { + key_type: record_class(py, RecordId::CompiledObjectKey)?, + resource_type: record_class(py, RecordId::CompiledResourceType)?, + keys: RefCell::new(HashMap::new()), + }) + } + + pub fn py(&self) -> Python<'py> { + self.key_type.py() + } + + /// The `CompiledObjectKey` for one `(resource type, name)` pair. + pub fn key(&self, (resource_type, name): &(String, String)) -> PyResult> { + let pair = (resource_type.clone(), name.clone()); + if let Some(existing) = self.keys.borrow().get(&pair) { + return Ok(existing.bind(self.py()).clone()); + } + let created: Bound<'py, PyAny> = self + .key_type + .call1((self.resource_type.call1((resource_type,))?, name))?; + self.keys + .borrow_mut() + .insert(pair, created.clone().unbind()); + Ok(created) + } + + /// `CompiledObjectKey`s for `pairs`, in order. + pub fn keys(&self, pairs: &[(String, String)]) -> PyResult>> { + pairs.iter().map(|pair| self.key(pair)).collect() + } +} + +/// Python keys a walk visited, compared with Python hashing and equality. +#[derive(Debug)] +pub struct PyNodeSet<'py>(pub Bound<'py, PySet>); + +impl<'py> NodeSet> for PyNodeSet<'py> { + type Error = pyo3::PyErr; + + fn insert_new(&mut self, key: &Bound<'py, PyAny>) -> Result { + if self.0.contains(key)? { + return Ok(false); + } + self.0.add(key)?; + Ok(true) + } + + fn holds(&self, key: &Bound<'py, PyAny>) -> Result { + self.0.contains(key) + } +} + +/// `(key, deps, tags, model folder)`. +#[derive(Debug)] +pub struct ResourceRow { + pub key: ObjectKey, + pub deps: Vec, + pub tags: Vec, + pub folder: Option, +} + +impl<'py> pyo3::FromPyObject<'py> for ResourceRow { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (key, deps, tags, folder): (ObjectKey, Vec, Vec, Option) = + pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + key, + deps, + tags, + folder, + }) + } +} + +/// A project graph: lineage edges, tag, path and name indexes, and selector resolution. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeProjectGraph { + pub(crate) graph: ProjectGraph, +} + +/// Static compiled project graph held natively; dict views are built only when read. +#[pyclass(name = "ProjectGraph", module = "sqlbuild._native", frozen)] +pub struct CompiledProjectGraph { + pub(crate) project: Py, + pub(crate) native: Py, + pub(crate) views: PyOnceLock>, +} + +impl std::fmt::Debug for CompiledProjectGraph { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CompiledProjectGraph") + .finish_non_exhaustive() + } +} + +impl NativeProjectGraph { + /// The graph this object wraps. + pub fn graph(&self) -> &ProjectGraph { + &self.graph + } + + /// Wrap a graph another native stage built, such as project assembly. + pub fn new(graph: ProjectGraph) -> Self { + Self { graph } + } +} + +impl CompiledProjectGraph { + /// The graph of `project` over its native lineage graph. + pub fn new(project: Py, native: Py) -> Self { + Self { + project, + native, + views: PyOnceLock::new(), + } + } + + /// The compiled project this graph indexes. + pub fn project<'py>(&self, py: Python<'py>) -> &Bound<'py, PyAny> { + self.project.bind(py) + } + + /// The native lineage graph. + pub fn native<'py>(&self, py: Python<'py>) -> &Bound<'py, NativeProjectGraph> { + self.native.bind(py) + } + + pub(crate) fn view(&self, py: Python<'_>, name: &str) -> PyResult> { + Ok(native_field(self.lineage_views(py)?.bind(py), name)?.unbind()) + } +} + +#[pymethods] +impl CompiledProjectGraph { + #[new] + #[pyo3(signature = (*, project, native))] + pub(crate) fn py_new(project: Py, native: Py) -> Self { + Self::new(project, native) + } + + #[getter(project)] + pub fn project_attribute(&self, py: Python<'_>) -> Py { + self.project.clone_ref(py) + } + + #[getter(native)] + pub fn native_attribute(&self, py: Python<'_>) -> Py { + self.native.clone_ref(py) + } + + /// Lineage edges and selector indexes, converted once. + #[getter] + pub fn lineage_views(&self, py: Python<'_>) -> PyResult> { + self.views + .get_or_try_init(py, || { + Ok(lineage_graph_views(py, self.native.bind(py).borrow())?.unbind()) + }) + .map(|views| views.clone_ref(py)) + } + + #[getter] + pub fn upstream_deps(&self, py: Python<'_>) -> PyResult> { + self.view(py, "upstream_deps") + } + + #[getter] + pub fn downstream_deps(&self, py: Python<'_>) -> PyResult> { + self.view(py, "downstream_deps") + } + + #[getter] + pub fn tag_index(&self, py: Python<'_>) -> PyResult> { + self.view(py, "tag_index") + } + + #[getter] + pub fn path_index(&self, py: Python<'_>) -> PyResult> { + self.view(py, "path_index") + } + + #[getter] + pub fn all_keys(&self, py: Python<'_>) -> PyResult> { + self.view(py, "all_keys") + } +} + +#[pymethods] +impl NativeProjectGraph { + /// Build the indexes from resources in project order: models, sources, seeds, functions. + #[staticmethod] + pub(crate) fn from_resources(resources: Vec) -> PyResult { + project_graph(resources) + } + + /// Wrap indexes a caller already built, such as the planner's. + #[staticmethod] + pub(crate) fn from_indexes(indexes: IndexesRow) -> PyResult { + compiler_guard(|| { + let IndexesRow { + names, + upstream, + downstream, + tags, + paths, + } = indexes; + Ok(Self::new(graph_from_indexes(GraphIndexes { + names, + upstream, + downstream, + tags, + paths, + }))) + }) + } + + /// `(names, upstream, downstream, tags, paths)` for Python's `ProjectGraph`. + pub(crate) fn indexes(&self) -> IndexesRow { + let graph: &ProjectGraph = &self.graph; + IndexesRow { + names: graph.names.clone(), + upstream: graph.upstream.clone(), + downstream: graph.downstream.clone(), + tags: graph.tags.clone(), + paths: graph.paths.clone(), + } + } + + /// The key's direct upstream lineage keys, in authored dependency order. + pub fn upstream(&self, key: ObjectKey) -> Vec { + edge_keys(&self.graph, &key, EdgeDirection::Upstream).to_vec() + } + + /// The key's direct downstream keys, sorted by `(resource type, name)`. + pub(crate) fn downstream(&self, key: ObjectKey) -> Vec { + edge_keys(&self.graph, &key, EdgeDirection::Downstream).to_vec() + } + + /// Every selectable key, in project order with later same-name resources replacing earlier. + pub(crate) fn keys(&self) -> Vec { + self.graph + .names + .iter() + .map(|(_, key)| key.clone()) + .collect() + } + + /// `(selector name, key)` in project order. + pub(crate) fn names(&self) -> Vec<(String, ObjectKey)> { + self.graph.names.clone() + } + + /// Every lineage edge as `(dependency, dependent)`, in upstream order. + pub(crate) fn edges(&self) -> Vec<(ObjectKey, ObjectKey)> { + self.graph + .upstream + .iter() + .flat_map(|(key, deps)| dependency_edges(key, deps)) + .collect() + } + + /// The manifest's `(parent_map, child_map)` entries as dbt-style unique ids, in graph order. + pub(crate) fn unique_id_maps( + &self, + project_name: &str, + prefixes: HashMap, + ) -> (UniqueIdMap, UniqueIdMap) { + ( + unique_id_edges( + &self.graph, + EdgeDirection::Upstream, + project_name, + &prefixes, + ), + unique_id_edges( + &self.graph, + EdgeDirection::Downstream, + project_name, + &prefixes, + ), + ) + } + + /// The compile report's execution layer count over models. + pub(crate) fn model_layer_count(&self) -> usize { + model_layer_count(&self.graph) + } + + /// `--select` minus `--exclude`, plus the functions they need to build. + pub(crate) fn resolve( + &self, + select: Vec, + exclude: Vec, + ) -> PyResult { + compiler_guard(|| Ok(outcome(resolve_selectors(&self.graph, &select, &exclude)))) + } + + /// The keys selector tokens match, without build resources. + pub(crate) fn tokens(&self, selectors: Vec) -> PyResult { + compiler_guard(|| Ok(outcome(resolve_selector_tokens(&self.graph, &selectors)))) + } + + /// The keys one `kind:value` selector names before `+` expansion. + pub(crate) fn matched(&self, kind: &str, value: &str) -> PyResult { + compiler_guard(|| Ok(outcome(match_selector(&self.graph, kind, value)))) + } + + /// The selected keys plus the functions and seeds Python adds to build them. + pub(crate) fn build_resources( + &self, + selected: Vec, + include: (bool, bool, bool), + ) -> PyResult> { + compiler_guard(|| { + let (upstream_functions, upstream_seeds, downstream_functions) = include; + Ok(build_resources( + &self.graph, + &selected, + BuildResources { + upstream_functions, + upstream_seeds, + downstream_functions, + }, + )) + }) + } +} + +/// A compiled project's sources and discovered Python nodes, which publication resolves selectors over. +#[derive(Debug)] +pub struct ProjectSelection { + pub(crate) sources: Vec<(String, Option)>, + pub nodes: PythonSelectionGraph, +} + +impl ProjectSelection { + pub fn new(project: &Bound<'_, PyAny>, discovered_inputs: &Bound<'_, PyAny>) -> PyResult { + Ok(Self { + sources: project_sources(project)?, + nodes: selection_graph(&python_node_graph(discovered_inputs)?)?, + }) + } + + /// The `kind:name` keys `sqb build --select` runs for these selectors. + pub fn run_keys( + &self, + graph: &ProjectGraph, + select: &[String], + exclude: &[String], + ) -> Result, String> { + self.keys(run_selection(self.sql(graph), &self.nodes, select, exclude)) + } + + /// The `kind:name` keys these selectors name, Python checks included. + pub fn node_keys( + &self, + graph: &ProjectGraph, + select: &[String], + exclude: &[String], + ) -> Result, String> { + self.keys(resolve_python_sql_nodes( + self.sql(graph), + &self.nodes, + select, + exclude, + )) + } + + /// The published key of a Python node, such as `task:export_orders`. + pub fn node_key(&self, name: &str) -> Option { + self.nodes + .nodes + .iter() + .rev() + .find(|node| node.name == name) + .map(|node| format!("{}:{name}", node.kind)) + } + + /// The key a selected Python node's run reports under; a loader reports as the source it writes. + pub(crate) fn run_key(&self, name: &str) -> Option { + let node = self + .nodes + .nodes + .iter() + .rev() + .find(|node| node.name == name)?; + Some(python_run_key(&self.sources, (&node.kind, name))) + } + + /// Whether a loader writes no declared source and so runs only as part of another loader's chain. + pub fn intermediate_loader(&self, name: &str) -> bool { + !self + .sources + .iter() + .any(|(_, loader)| loader.as_deref() == Some(name)) + } + + pub(crate) fn sql<'graph>( + &'graph self, + graph: &'graph ProjectGraph, + ) -> SqlSelectionGraph<'graph> { + SqlSelectionGraph { + graph, + sources: &self.sources, + } + } + + pub(crate) fn keys( + &self, + selection: Result, + ) -> Result, String> { + let selection: RunSelection = selection.map_err(|error| error.message)?; + let mut keys: BTreeSet = selection + .sql_keys + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + for name in &selection.python_node_names { + keys.extend(self.run_key(name)); + } + Ok(keys) + } +} + +/// Python's selector name, edge, tag and path mappings. +#[derive(Debug)] +pub struct GraphViewMappings<'a, 'py> { + pub all_keys: &'a Bound<'py, PyAny>, + pub upstream: &'a Bound<'py, PyAny>, + pub downstream: &'a Bound<'py, PyAny>, + pub tag_index: &'a Bound<'py, PyAny>, + pub path_index: &'a Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python-project/src/lib.rs b/crates/sqlbuild-python-project/src/lib.rs new file mode 100644 index 0000000000..7af797d10d --- /dev/null +++ b/crates/sqlbuild-python-project/src/lib.rs @@ -0,0 +1,27 @@ +//! The project layer of the `sqlbuild._native` bindings: the compiled project and its views, graph, references, lineage, fingerprints, semantic shapes, planner scope, selection and strategy, and SQL test planning. + +#![forbid(unsafe_code)] + +pub mod analysis; +pub mod analysis_session; +pub mod auditing; +pub mod compile_attachment; +pub mod compiled_project; +pub mod compiled_views; +pub mod contracts; +pub mod fingerprints; +pub mod graph; +pub mod lineage; +pub mod native_host; +pub mod planner_identity; +pub mod planner_phases; +pub mod planner_scope; +pub mod planner_selection; +pub mod planner_strategy; +pub mod project_snapshot; +pub mod references; +pub mod relation_naming; +pub mod resource_views; +pub mod semantic_shapes; +pub mod sql_test_glue; +pub mod type_system; diff --git a/crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs b/crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs new file mode 100644 index 0000000000..caf95780e4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs @@ -0,0 +1,108 @@ +//! Sequence behaviour of the natively hosted `CompactLineageFacts` record. + +use pyo3::exceptions::PyIndexError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; +use sqlbuild_python_base::positional_record; + +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_base::records::types::RecordId; + +/// `ColumnTransformKind` members by native transform code. +pub(crate) const TRANSFORM_KINDS: [&str; 6] = [ + "DIRECT", + "CAST", + "EXPRESSION", + "AGGREGATION", + "STAR", + "CONSTANT", +]; +/// `ColumnLineageConfidence` members by native confidence code. +pub(crate) const CONFIDENCES: [&str; 3] = ["UNKNOWN", "HIGH", "MEDIUM"]; + +/// The enum member at `code` of `members`. +pub(crate) fn coded_member( + py: Python<'_>, + (class, members): (RecordId, &[&str]), + code: usize, +) -> PyResult> { + let Some(member) = members.get(code) else { + return Err(PyIndexError::new_err("tuple index out of range")); + }; + Ok(enum_member(py, class, member)?.unbind()) +} + +/// `CompactLineageFacts.resource_name(index)`: the model's name for a native relation index. +pub(crate) fn resource_name<'py>( + facts: &Bound<'py, PyAny>, + index: &Bound<'py, PyAny>, +) -> PyResult> { + let names: Bound<'py, PyAny> = native_field(facts, "resource_names")?; + let name: Bound<'py, PyAny> = mapping_get(&names, index)?; + if !name.is_none() { + return Ok(name); + } + let pooled: Bound<'py, PyAny> = + native_field(facts, "resource_name_indexes")?.call_method1("get", (index, index))?; + native_field(facts, "string_pool")?.get_item(pooled) +} + +positional_record! { + extract; + /// One compact lineage row: output name index, transform and confidence codes, sources. + struct CompactColumnRow<'py> { + name_index: Bound<'py, PyAny>, + transform_code: usize, + confidence_code: usize, + sources: Bound<'py, PyAny>, + } +} + +/// One compact row projected to a `CompiledLineageColumnFact`. +pub(crate) fn project_row(facts: &Bound<'_, PyAny>, row: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = facts.py(); + let pool: Bound<'_, PyAny> = native_field(facts, "string_pool")?; + let CompactColumnRow { + name_index, + transform_code, + confidence_code, + sources, + }: CompactColumnRow<'_> = row.extract()?; + let mut upstream: Vec> = Vec::new(); + for source in sources.try_iter()? { + let source: Bound<'_, PyTuple> = source?.cast_into::()?; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("resource_type", pool.get_item(source.get_item(0)?)?)?; + keywords.set_item("resource_name", resource_name(facts, &source.get_item(1)?)?)?; + keywords.set_item("column_name", pool.get_item(source.get_item(2)?)?)?; + upstream.push(new_record( + py, + RecordId::CompiledLineageSourceFact, + (), + Some(&keywords), + )?); + } + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("output_column", pool.get_item(name_index)?)?; + keywords.set_item("upstream_columns", PyTuple::new(py, upstream)?)?; + keywords.set_item( + "transform_kind", + coded_member( + py, + (RecordId::ColumnTransformKind, &TRANSFORM_KINDS), + transform_code, + )?, + )?; + keywords.set_item( + "confidence", + coded_member( + py, + (RecordId::ColumnLineageConfidence, &CONFIDENCES), + confidence_code, + )?, + )?; + Ok(new_record(py, RecordId::CompiledLineageColumnFact, (), Some(&keywords))?.unbind()) +} diff --git a/crates/sqlbuild-python-project/src/lineage/_helpers/mod.rs b/crates/sqlbuild-python-project/src/lineage/_helpers/mod.rs new file mode 100644 index 0000000000..a662b6c8bd --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod lineage_facts; diff --git a/crates/sqlbuild-python-project/src/lineage/main/confidence_member.rs b/crates/sqlbuild-python-project/src/lineage/main/confidence_member.rs new file mode 100644 index 0000000000..3adb6c3356 --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/confidence_member.rs @@ -0,0 +1,11 @@ +//! The `ColumnLineageConfidence` member of a native confidence code. + +use crate::lineage::_helpers::lineage_facts::CONFIDENCES; +use crate::lineage::_helpers::lineage_facts::coded_member; +use pyo3::prelude::{Py, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::types::RecordId; + +/// The `ColumnLineageConfidence` member of a native confidence code. +pub fn confidence_member(py: Python<'_>, code: usize) -> PyResult> { + coded_member(py, (RecordId::ColumnLineageConfidence, &CONFIDENCES), code) +} diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_eq.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_eq.rs new file mode 100644 index 0000000000..8121e58eb4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_eq.rs @@ -0,0 +1,28 @@ +//! `facts == other`: item-wise equality with any sequence. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple, PyTupleMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// `facts == other`: item-wise equality with any sequence. +pub fn facts_eq( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = instance.py(); + let other: Bound<'_, PyAny> = args.get_item(0)?; + let sequence: Bound<'_, PyAny> = py.import("collections.abc")?.getattr("Sequence")?; + if !other.is_instance(&sequence)? { + return Ok(py.NotImplemented()); + } + let mine: Bound<'_, PyAny> = native_field(instance, "facts")?; + let theirs: Bound<'_, PyTuple> = PyTuple::new( + py, + other + .try_iter()? + .collect::>>>()?, + )?; + mine.eq(theirs)?.into_py_any(py) +} diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_getitem.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_getitem.rs new file mode 100644 index 0000000000..9c214bbf3d --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_getitem.rs @@ -0,0 +1,30 @@ +//! `facts[index]`: one fact, or a tuple of facts for a slice. + +use pyo3::exceptions::PyIndexError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PySlice, PyTuple, PyTupleMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// `facts[index]`: one fact, or a tuple of facts for a slice. +pub fn facts_getitem( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let index: Bound<'_, PyAny> = args.get_item(0)?; + let projected: Bound<'_, PyAny> = native_field(instance, "facts")?; + if index.is_instance_of::() { + return Ok(projected.get_item(index)?.unbind()); + } + let length: isize = isize::try_from(projected.len()?)?; + let requested: isize = index.extract()?; + let normalized: isize = if requested >= 0 { + requested + } else { + length + requested + }; + if normalized < 0 || normalized >= length { + return Err(PyIndexError::new_err(index.unbind())); + } + Ok(projected.get_item(normalized)?.unbind()) +} diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_iter.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_iter.rs new file mode 100644 index 0000000000..0c7a7a09cd --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_iter.rs @@ -0,0 +1,17 @@ +//! `iter(facts)`. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// `iter(facts)`. +pub fn facts_iter( + instance: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + Ok(native_field(instance, "facts")? + .try_iter()? + .into_any() + .unbind()) +} diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_len.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_len.rs new file mode 100644 index 0000000000..507ce9ea26 --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_len.rs @@ -0,0 +1,17 @@ +//! `len(facts)`. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// `len(facts)`. +pub fn facts_len( + instance: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + native_field(instance, "rows")? + .len()? + .into_py_any(instance.py()) +} diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs new file mode 100644 index 0000000000..06306bbecc --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs @@ -0,0 +1,15 @@ +//! `CompactLineageFacts.facts`: every row projected once, in order. + +use crate::lineage::_helpers::lineage_facts::project_row; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyTuple; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// `CompactLineageFacts.facts`: every row projected once, in order. +pub fn facts_projected(instance: &Bound<'_, PyAny>) -> PyResult> { + let projected: Vec> = native_field(instance, "rows")? + .try_iter()? + .map(|row| project_row(instance, &row?)) + .collect::>()?; + Ok(PyTuple::new(instance.py(), projected)?.into_any().unbind()) +} diff --git a/crates/sqlbuild-python-project/src/lineage/main/facts_resource_name.rs b/crates/sqlbuild-python-project/src/lineage/main/facts_resource_name.rs new file mode 100644 index 0000000000..a94ef72be9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/facts_resource_name.rs @@ -0,0 +1,16 @@ +//! `CompactLineageFacts.resource_name(index)`. + +use crate::lineage::_helpers::lineage_facts::resource_name; +use pyo3::prelude::{Bound, Py, PyAny, PyResult}; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_python_base::boundary::main::call_argument::call_argument; + +/// `CompactLineageFacts.resource_name(index)`. +pub fn facts_resource_name( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let index: Bound<'_, PyAny> = call_argument((args, kwargs), "index", 0)?; + Ok(resource_name(instance, &index)?.unbind()) +} diff --git a/crates/sqlbuild-python-project/src/lineage/main/mod.rs b/crates/sqlbuild-python-project/src/lineage/main/mod.rs new file mode 100644 index 0000000000..0f7319858c --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/mod.rs @@ -0,0 +1,8 @@ +pub mod confidence_member; +pub mod facts_eq; +pub mod facts_getitem; +pub mod facts_iter; +pub mod facts_len; +pub mod facts_projected; +pub mod facts_resource_name; +pub mod transform_kind_member; diff --git a/crates/sqlbuild-python-project/src/lineage/main/transform_kind_member.rs b/crates/sqlbuild-python-project/src/lineage/main/transform_kind_member.rs new file mode 100644 index 0000000000..0d4671f8df --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/main/transform_kind_member.rs @@ -0,0 +1,11 @@ +//! The `ColumnTransformKind` member of a native transform code. + +use crate::lineage::_helpers::lineage_facts::TRANSFORM_KINDS; +use crate::lineage::_helpers::lineage_facts::coded_member; +use pyo3::prelude::{Py, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::types::RecordId; + +/// The `ColumnTransformKind` member of a native transform code. +pub fn transform_kind_member(py: Python<'_>, code: usize) -> PyResult> { + coded_member(py, (RecordId::ColumnTransformKind, &TRANSFORM_KINDS), code) +} diff --git a/crates/sqlbuild-python-project/src/lineage/mod.rs b/crates/sqlbuild-python-project/src/lineage/mod.rs new file mode 100644 index 0000000000..009b7f7a66 --- /dev/null +++ b/crates/sqlbuild-python-project/src/lineage/mod.rs @@ -0,0 +1,4 @@ +//! Lineage facts of compiled models. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs new file mode 100644 index 0000000000..b336f64d4d --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod orchestration_triggers; diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/orchestration_triggers.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/orchestration_triggers.rs new file mode 100644 index 0000000000..1f001e8d52 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/orchestration_triggers.rs @@ -0,0 +1,156 @@ +//! Authored trigger controls and bounded calendars evaluated only in the location interpreter. + +use std::collections::BTreeSet; + +use chrono::{DateTime, TimeDelta, Utc}; +use chrono_tz::Tz; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use serde_json::{Value, json}; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; +use sqlbuild_orchestration::declarations::models::Schedule; +use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; + +use crate::graph::models::ProjectSelection; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::native_host::constants::CALENDAR_WINDOW_PARAMETER; +use sqlbuild_python_base::native_host::constants::SOURCE_NEW_DATA_CONDITION; +use sqlbuild_python_base::native_host::constants::SOURCE_STALE_CONDITION; +use sqlbuild_python_front::discovery::main::python_files::provider_usages::provider_usages; + +fn horizon(value: &str) -> PyResult { + let invalid = || { + PyValueError::new_err( + "calendar horizon must be a positive fixed duration using d, h, m or s, within datetime range", + ) + }; + let digits = parse_duration(value).ok_or_else(invalid)?; + if [&digits.years, &digits.months] + .iter() + .any(|digits| !digits.trim_start_matches('0').is_empty()) + { + return Err(invalid()); + } + let seconds = [ + (digits.days, 86_400_i64), + (digits.hours, 3_600), + (digits.minutes, 60), + (digits.seconds, 1), + ] + .into_iter() + .try_fold(0_i64, |total, (digits, factor)| { + let digits = digits.parse::().map_err(|_| invalid())?; + let seconds = digits.checked_mul(factor).ok_or_else(invalid)?; + total.checked_add(seconds).ok_or_else(invalid) + })?; + TimeDelta::try_seconds(seconds).ok_or_else(invalid) +} + +pub(crate) fn calendar_times( + (authored, timezone): (&Bound<'_, PyAny>, &str), + session: &Bound<'_, PyAny>, + providers: &Bound<'_, PyDict>, + start: DateTime, +) -> PyResult { + let py = authored.py(); + timezone + .parse::() + .map_err(|_| PyValueError::new_err("calendar timezone must be an IANA timezone"))?; + let duration = horizon(&authored.getattr("horizon")?.extract::()?)?; + let end = start + .checked_add_signed(duration) + .ok_or_else(|| PyValueError::new_err("calendar horizon is out of datetime range"))?; + let datetime = imported(py, "datetime", "datetime")?; + let zone = imported(py, "zoneinfo", "ZoneInfo")?.call1((timezone,))?; + let window = PyDict::new(py); + for (name, instant) in [("start", start), ("end", end)] { + window.set_item( + name, + datetime + .call_method1("fromisoformat", (instant.to_rfc3339(),))? + .call_method1("astimezone", (&zone,))?, + )?; + } + let window = py + .import("sqlbuild.orchestration")? + .getattr("CalendarWindow")? + .call((), Some(&window))?; + let function = authored.getattr("function")?; + let arguments = PyDict::new(py); + arguments.set_item(CALENDAR_WINDOW_PARAMETER, window)?; + for usage in provider_usages(&function, providers)?.try_iter()? { + let usage = usage?; + if usage.getattr("parameter_name")?.extract::()? == CALENDAR_WINDOW_PARAMETER { + continue; + } + arguments.set_item( + usage.getattr("parameter_name")?, + session.call_method1("get", (usage.getattr("provider_name")?,))?, + )?; + } + let mut times: BTreeSet> = BTreeSet::new(); + for (index, time) in function + .call((), Some(&arguments))? + .try_iter()? + .take(MAX_TICKS_PER_BATCH + 1) + .enumerate() + { + if index == MAX_TICKS_PER_BATCH { + return Err(PyValueError::new_err(format!( + "calendar exceeds {MAX_TICKS_PER_BATCH} upcoming times" + ))); + } + let time = time?; + if !time.is_instance(&datetime)? { + return Err(PyValueError::new_err( + "calendar times must be timezone-aware datetimes", + )); + } + let time = + DateTime::parse_from_rfc3339(&time.call_method0("isoformat")?.extract::()?) + .map_err(|_| { + PyValueError::new_err("calendar times must be timezone-aware datetimes") + })? + .with_timezone(&Utc); + if time < start || time >= end { + return Err(PyValueError::new_err( + "calendar time lies outside its half-open evaluation window", + )); + } + times.insert(time); + } + Ok(Schedule::Calendar { + times, + refresh_at: start + duration / 2, + }) +} + +pub(crate) fn sensor_nodes( + (graph, selected): (&ProjectGraph, &ProjectSelection), + selectors: &[String], + field: &str, +) -> PyResult { + if selectors.is_empty() || selectors.iter().any(|value| value.trim().is_empty()) { + return Err(PyValueError::new_err(format!( + "{field} needs nonempty selectors" + ))); + } + let nodes: BTreeSet = selected + .node_keys(graph, selectors, &[]) + .map_err(|error| PyValueError::new_err(format!("{field}: {error}")))?; + if nodes.is_empty() { + return Err(PyValueError::new_err(format!( + "{field} needs nonempty selectors" + ))); + } + if matches!(field, SOURCE_NEW_DATA_CONDITION | SOURCE_STALE_CONDITION) + && nodes.iter().any(|key| !key.starts_with("source:")) + { + return Err(PyValueError::new_err(format!( + "{field} must select only source nodes" + ))); + } + Ok(json!(nodes)) +} diff --git a/crates/sqlbuild-python-project/src/native_host/main/compile_calendars.rs b/crates/sqlbuild-python-project/src/native_host/main/compile_calendars.rs new file mode 100644 index 0000000000..19194f7922 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/main/compile_calendars.rs @@ -0,0 +1,83 @@ +//! Each calendar request's trigger, with each (calendar, timezone) pair evaluated once. + +use crate::native_host::_helpers::orchestration_triggers::calendar_times; +use crate::native_host::models::CalendarRequest; +use chrono::{Timelike, Utc}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyBool; +use sqlbuild_host::entries::models::UserCodeHost; +use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::native_host::main::host_entry::host_entry as entry; +use sqlbuild_python_front::discovery::main::python_files::discover_provider_classes::discover_provider_classes; +use sqlbuild_python_front::discovery::main::python_files::provider_by_name::provider_by_name; +use sqlbuild_python_front::discovery::models::SecretPolicy; +use std::collections::BTreeMap; +use std::path::Path; + +/// Each calendar request's trigger, with each (calendar, timezone) pair evaluated once. +pub fn compile_calendars( + py: Python<'_>, + project_dir: &Path, + (calendars, requests): (&BTreeMap>, Vec), + mut definitions: Definitions, +) -> PyResult { + if requests.is_empty() { + return Ok(definitions); + } + let providers = discover_provider_classes( + imported(py, "pathlib", "Path")?.call1((project_dir,))?, + SecretPolicy::Defer.name(), + )?; + let by_name = provider_by_name(&providers)?; + let session = call_with( + &entry(py, UserCodeHost::BuildProviderSession)?, + &[ + ("discovered_providers", providers), + ("allow_shared", PyBool::new(py, false).to_owned().into_any()), + ], + )?; + let now = Utc::now(); + let start = now + .with_nanosecond(now.timestamp_subsec_micros() * 1_000) + .ok_or_else(|| PyValueError::new_err("calendar clock is out of range"))?; + within(&session, |_| { + let mut evaluated: BTreeMap<(String, String), Schedule> = BTreeMap::new(); + for request in requests { + let key = (request.calendar.clone(), request.timezone.clone()); + let schedule = match evaluated.get(&key) { + Some(schedule) => schedule.clone(), + None => { + let calendar = calendars.get(&request.calendar).ok_or_else(|| { + PyValueError::new_err(format!( + "calendar '{}' is not declared by a calendar module", + request.calendar + )) + })?; + let schedule = + calendar_times((calendar, &request.timezone), &session, &by_name, start) + .map_err(|error| { + PyValueError::new_err(format!( + "calendar '{}': {error}", + request.calendar + )) + })?; + evaluated.insert(key, schedule.clone()); + schedule + } + }; + let mut trigger = TriggerDefinition { + schedule, + ..request.trigger + }; + trigger.starts_at.get_or_insert(start); + definitions + .triggers + .insert(trigger.job_key.clone(), trigger); + } + Ok(definitions) + }) +} diff --git a/crates/sqlbuild-python-project/src/native_host/main/compile_sensors.rs b/crates/sqlbuild-python-project/src/native_host/main/compile_sensors.rs new file mode 100644 index 0000000000..f61aef685c --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/main/compile_sensors.rs @@ -0,0 +1,80 @@ +//! Compile sensors. + +use crate::graph::models::ProjectSelection; +use crate::native_host::_helpers::orchestration_triggers::sensor_nodes; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{PyResult, Python}; +use serde_json::json; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_orchestration::declarations::models::Definitions; +use sqlbuild_orchestration::sensors::models::SensorDefinition; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use sqlbuild_python_base::native_host::constants::SOURCE_NEW_DATA_CONDITION; +use sqlbuild_python_base::native_host::constants::SOURCE_STALE_CONDITION; +use sqlbuild_python_base::orchestration_reads::main::callback_code_hash::callback_code_hash; +use sqlbuild_python_base::orchestration_reads::main::declared_at::declared_at; +use sqlbuild_python_base::orchestration_reads::main::orchestration_code_hash::orchestration_code_hash; +use sqlbuild_python_base::orchestration_reads::models::AuthoredSensor; +use std::path::Path; + +/// Compile sensors. +pub fn compile_sensors( + py: Python<'_>, + (project_dir, input, graph): (&Path, &SnapshotInput, (&ProjectGraph, &ProjectSelection)), + authored: Vec, + mut definitions: Definitions, +) -> PyResult { + let sources = if authored.iter().any(|sensor| sensor.function.is_some()) { + orchestration_code_hash(project_dir)? + } else { + String::new() + }; + for sensor in authored { + if !sensor.targets.is_empty() && !sensor.targets.iter().any(|value| value == &input.target) + { + continue; + } + let condition = match (sensor.when, sensor.function) { + (Some(when), None) => { + let mut condition = json!({"kind": when.kind}); + if when.kind == SOURCE_STALE_CONDITION { + condition["after_seconds"] = when.after_seconds; + } else if when.kind != SOURCE_NEW_DATA_CONDITION { + condition["kinds"] = json!(when.kinds); + } + if let Some(job) = when.job { + condition["job_key"] = json!(job); + } + if let Some(select) = when.select { + let selectors: Vec = + select.split_whitespace().map(str::to_owned).collect(); + condition["selection"] = sensor_nodes(graph, &selectors, &when.kind)?; + } + if let Some(selectors) = when.executed_successfully { + condition["executed_successfully"] = + sensor_nodes(graph, &selectors, "executed_successfully")?; + } + condition + } + (None, Some(function)) => json!({"kind": "custom", "function": sensor.name, + "code_hash": callback_code_hash(function.bind(py), &sources)?}), + _ => { + return Err(PyValueError::new_err( + "sensor needs either a built-in condition or a decorated polling callback; callbacks with when are not supported yet", + )); + } + }; + let definition: SensorDefinition = serde_json::from_value(json!({ + "description": sensor.description, + "job_key": sensor.run, + "when": condition, + "minimum_interval_seconds": sensor.minimum_interval_seconds, + "default_status": sensor.default_status, + "targets": sensor.targets, + "declared_at": declared_at(project_dir, sensor.declared_at)?, + })) + .map_err(|error| PyValueError::new_err(format!("sensor {}: {error}", sensor.name)))?; + definitions.sensors.insert(sensor.name, definition); + } + Ok(definitions) +} diff --git a/crates/sqlbuild-python-project/src/native_host/main/mod.rs b/crates/sqlbuild-python-project/src/native_host/main/mod.rs new file mode 100644 index 0000000000..9d80042f90 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/main/mod.rs @@ -0,0 +1,2 @@ +pub mod compile_calendars; +pub mod compile_sensors; diff --git a/crates/sqlbuild-python-project/src/native_host/mod.rs b/crates/sqlbuild-python-project/src/native_host/mod.rs new file mode 100644 index 0000000000..9fbdf0b3ef --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/mod.rs @@ -0,0 +1,5 @@ +//! Model lookup and orchestration triggers. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/model_lookup.rs b/crates/sqlbuild-python-project/src/native_host/models.rs similarity index 66% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/model_lookup.rs rename to crates/sqlbuild-python-project/src/native_host/models.rs index 78f7958d10..415afdd71a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/model_lookup.rs +++ b/crates/sqlbuild-python-project/src/native_host/models.rs @@ -1,37 +1,40 @@ -//! Public model-map inputs and projections around the native planner's model lookup. - -use std::sync::Arc; +//! Public records of native host. +use crate::compiled_views::models::NativeCompiledModel; +use crate::planner_scope::models::ScopeData; use pyo3::IntoPyObject; -use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; - -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::planner_scope::retained::ScopeData; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_orchestration::declarations::models::TriggerDefinition; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use std::sync::Arc; /// Native compiled views stay typed; caller-authored protocol implementations remain public values. #[derive(Debug)] -pub(crate) enum ModelHandle { +pub enum ModelHandle { View(Py), Public(Py), } impl ModelHandle { - pub(crate) fn read(value: &Bound<'_, PyAny>) -> Self { + pub fn read(value: &Bound<'_, PyAny>) -> Self { match value.cast::() { Ok(value) => Self::View(value.clone().unbind()), Err(_) => Self::Public(value.clone().unbind()), } } - pub(crate) fn bind<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { + pub fn bind<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { match self { Self::View(value) => value.bind(py).clone().into_any(), Self::Public(value) => value.bind(py).clone(), } } - pub(crate) fn clone_ref(&self, py: Python<'_>) -> Self { + pub fn clone_ref(&self, py: Python<'_>) -> Self { match self { Self::View(value) => Self::View(value.clone_ref(py)), Self::Public(value) => Self::Public(value.clone_ref(py)), @@ -41,20 +44,20 @@ impl ModelHandle { /// A public caller's mapping, or the compiler-owned models shared by native planning. #[derive(Debug)] -pub(crate) enum ModelLookup { +pub enum ModelLookup { Public(Py), Native(Arc), } impl ModelLookup { - pub(crate) fn clone_ref(&self, py: Python<'_>) -> Self { + pub fn clone_ref(&self, py: Python<'_>) -> Self { match self { Self::Public(value) => Self::Public(value.clone_ref(py)), Self::Native(value) => Self::Native(value.clone()), } } - pub(crate) fn get<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + pub fn get<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { match self { Self::Public(value) => mapping_get(value.bind(py), name), Self::Native(value) => Ok(value @@ -63,7 +66,7 @@ impl ModelLookup { } } - pub(crate) fn project<'py>(&self, py: Python<'py>) -> PyResult> { + pub fn project<'py>(&self, py: Python<'py>) -> PyResult> { match self { Self::Public(value) => Ok(value.bind(py).clone()), Self::Native(value) => value.field(py, "models_by_name"), @@ -86,3 +89,11 @@ impl<'py> IntoPyObject<'py> for &ModelLookup { self.project(py) } } + +/// A job's or group's schedule on a named calendar, as its trigger before the calendar's times fill it. +#[derive(Debug)] +pub struct CalendarRequest { + pub calendar: String, + pub timezone: String, + pub trigger: TriggerDefinition, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs similarity index 59% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs rename to crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs index a90cb8ea1e..aacdb803c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs @@ -1,36 +1,25 @@ //! Planner execution edges, their injected origins, stable execution order and buildability over compiled keys. +use crate::planner_identity::main::execution_deps::execution_deps; +use crate::planner_identity::main::execution_origins::execution_origins; +use crate::planner_identity::main::graph_key::graph_key; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; use std::collections::{HashMap, HashSet}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySetMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_analysis::graph::main::edge_keys::edge_keys; -use sqlbuild_analysis::graph::models::EdgeDirection; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; -use sqlbuild_analysis::planner_identity::main::execution_upstream::execution_upstream; use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message as message_of; use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; use sqlbuild_analysis::planner_identity::main::topological_order::topological_order; -use sqlbuild_analysis::planner_identity::models::{ - AvailableUpstreams, ExecutionTest, GraphKey, MissingUpstream, -}; +use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream}; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; -use crate::bindings::_helpers::graph::algorithms::{closure, inverted}; -use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::graph::views::edge_dict; -use crate::bindings::_helpers::planner_identity::hashing::graph_key; -use crate::bindings::_helpers::references::explicit_references::{ - GateRow, HookRow, native_execution_gate_edges, -}; +use crate::graph::main::nodes::closure::closure; +use crate::graph::main::nodes::inverted::inverted; +use crate::graph::main::project::edge_dict::edge_dict; +use crate::graph::models::KeyCache; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -39,7 +28,7 @@ use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; -const TABLE_FN_TEST_MODE: &str = "table_fn"; +pub(crate) const TABLE_FN_TEST_MODE: &str = "table_fn"; const INLINE_SQL_HOOK: &str = "inline SQL"; /// Python keys by graph key, keeping the first object seen for each. @@ -76,7 +65,7 @@ impl<'py> KeyObjects<'py> { } } -fn hook_label(hook_name: &str, kind: &str) -> String { +pub(crate) fn hook_label(hook_name: &str, kind: &str) -> String { if kind == INLINE_SQL_HOOK { return format!("inline SQL hook {hook_name}"); } @@ -91,45 +80,6 @@ pub(crate) fn build_execution_upstream_deps<'py>( edge_dict(&KeyCache::new(project.py())?, &execution_deps(project)?) } -/// Execution dependencies kept native until a public graph or plan asks for its dict view. -pub(crate) fn execution_deps( - project: &Bound<'_, PyAny>, -) -> PyResult)>> { - let mut resources: Vec<(ObjectKey, Vec)> = Vec::new(); - let collections: [Bound<'_, PyAny>; 4] = [ - project_reads::models(project)?, - project_reads::sources(project)?, - project_reads::seeds(project)?, - project_reads::functions(project)?, - ]; - for collection in collections { - for resource in collection.try_iter()? { - let row = resource_reads::graph_row(&resource?)?; - resources.push((row.key, row.deps)); - } - } - let mut tests: Vec = Vec::new(); - for test in project_reads::sql_tests(project)?.try_iter()? { - let test = test?; - tests.push(ExecutionTest { - key: native_key(&native_field(&test, "key")?)?, - scope_deps: native_field(&test, "scope_deps")? - .try_iter()? - .map(|key| native_key(&key?)) - .collect::>()?, - table_function: native_field(&test, "mode")?.str()?.extract::()? - == TABLE_FN_TEST_MODE, - }); - } - let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; - let injected: Vec<_> = gates - .into_iter() - .map(|gate| (gate.gated, gate.read)) - .chain(hooks.into_iter().map(|hook| (hook.gated, hook.read))) - .collect(); - Ok(execution_upstream(resources, &tests, &injected)) -} - /// Human-readable origins for execution edges injected beyond pure lineage. #[pyfunction] pub(crate) fn build_execution_edge_origins<'py>( @@ -144,95 +94,6 @@ pub(crate) fn build_execution_edge_origins<'py>( Ok(result) } -/// Injected-edge descriptions remain native for cycle and buildability diagnostics. -pub(crate) fn execution_origins( - project: &Bound<'_, PyAny>, -) -> PyResult> { - let mut origins: Vec<((ObjectKey, ObjectKey), String)> = Vec::new(); - for test in project_reads::sql_tests(project)?.try_iter()? { - let test = test?; - let test_name: String = native_field(&test, "name")?.extract()?; - for target_key in native_field(&test, "scope_deps")?.try_iter()? { - let target_key = native_key(&target_key?)?; - let target_name = &target_key.1; - let description = format!("SQL test '{test_name}' runs before '{target_name}'"); - origins.push(( - (target_key, native_key(&native_field(&test, "key")?)?), - description, - )); - } - } - let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; - let lineage = native_field(project, "lineage_graph")?; - let lineage = lineage.cast::()?.borrow(); - let is_lineage = |gated: &ObjectKey, read: &ObjectKey| { - edge_keys(lineage.graph(), gated, EdgeDirection::Upstream).contains(read) - }; - let mut seen: HashSet<_> = origins.iter().map(|(edge, _)| edge.clone()).collect(); - for GateRow { - audit_name, - target, - gated, - read, - } in &gates - { - if is_lineage(gated, read) || !seen.insert((gated.clone(), read.clone())) { - continue; - } - origins.push(( - (gated.clone(), read.clone()), - format!("audit '{audit_name}' on '{}' reads '{}'", target.1, read.1), - )); - } - for HookRow { - hook_name, - gated, - read, - hook_type: kind, - .. - } in &hooks - { - if is_lineage(gated, read) || !seen.insert((gated.clone(), read.clone())) { - continue; - } - origins.push(( - (gated.clone(), read.clone()), - format!( - "{} on '{}' reads '{}'", - hook_label(hook_name, kind), - gated.1, - read.1 - ), - )); - } - Ok(origins) -} - -/// Stable native execution order, projecting no Python graph or key objects. -pub(crate) fn execution_order( - py: Python<'_>, - upstream: &[(ObjectKey, Vec)], - origins: &[((ObjectKey, ObjectKey), String)], -) -> PyResult> { - let key = |(kind, name): &ObjectKey| GraphKey { - kind: kind.clone(), - name: name.clone(), - }; - let edges: Vec<_> = upstream - .iter() - .map(|(node, deps)| (key(node), deps.iter().map(key).collect())) - .collect(); - let origins: HashMap<(GraphKey, GraphKey), String> = origins - .iter() - .map(|((gated, read), origin)| ((key(gated), key(read)), origin.clone())) - .collect(); - Ok(topological_order(&edges, &origins) - .map_err(|message| planner_input_error(py, message))? - .into_iter() - .map(|key| (key.kind, key.name)) - .collect()) -} - /// Downstream edges keyed by upstream key, each list sorted by `(resource_type, name)`. #[pyfunction] pub(crate) fn build_downstream_deps<'py>( @@ -257,7 +118,7 @@ pub(crate) fn build_downstream_deps<'py>( Ok(result) } -fn planner_input_error(py: Python<'_>, message: String) -> PyErr { +pub(crate) fn planner_input_error(py: Python<'_>, message: String) -> PyErr { new_plain_error(py, ErrorId::PlannerInput, &message) } @@ -411,16 +272,3 @@ pub(crate) fn missing_upstream_message<'py>( self::edge_origins(edge_origins, &mut objects)?; Ok(PyString::new(missing.py(), &message_of(&rows, &origins))) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_execution_upstream_deps, module)?)?; - module.add_function(wrap_pyfunction!(build_execution_edge_origins, module)?)?; - module.add_function(wrap_pyfunction!(build_downstream_deps, module)?)?; - module.add_function(wrap_pyfunction!(topologically_order_keys, module)?)?; - module.add_function(wrap_pyfunction!( - build_downstream_model_name_closure, - module - )?)?; - module.add_function(wrap_pyfunction!(check_buildability, module)?)?; - module.add_function(wrap_pyfunction!(missing_upstream_message, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs similarity index 50% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs rename to crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs index 2b4ad21b55..6dd300ebc1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs @@ -1,16 +1,11 @@ //! Query and function definition fingerprints for change detection, failing as planner input errors. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyString; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_identity::constants::{FINGERPRINT_HELP, QUERY_FINGERPRINT_FAILED}; -use sqlbuild_analysis::planner_identity::models::GraphKey; use sqlbuild_analysis::sql_tokens::main::baseline_fingerprint::baseline_fingerprint; use sqlbuild_analysis::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; -use sqlbuild_analysis::sql_tokens::main::definition_fingerprint::definition_fingerprint; use sqlbuild_core::panics::main::is_compiler_panic::is_compiler_panic; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; @@ -30,35 +25,8 @@ pub(crate) fn model_query_hash( .map_err(|error| fingerprint_error(py, &format!("Model '{model_name}' SQL"), error)) } -/// The model's query fingerprint, or its recorded one while the recorded SQL still fingerprints the same. -pub(crate) fn baselined_model_query_hash( - py: Python<'_>, - model_name: &str, - (query_sql, dialect): (&str, Option<&str>), - recorded: &Bound<'_, PyAny>, -) -> PyResult { - let current: String = model_query_hash(py, model_name, (query_sql, dialect))?; - recorded_baseline(py, current, recorded, |sql| { - compiler_query_fingerprint(sql, dialect) - }) -} - -/// The function's definition fingerprint, or its recorded one while the recorded definition still fingerprints the same. -pub(crate) fn baselined_function_definition_hash( - py: Python<'_>, - function_name: &str, - (fingerprint_sql, language, dialect): (&str, &str, Option<&str>), - recorded: &Bound<'_, PyAny>, -) -> PyResult { - let current: String = - function_definition_hash(py, function_name, (fingerprint_sql, language, dialect))?; - recorded_baseline(py, current, recorded, |sql| { - definition_fingerprint(sql, language, dialect) - }) -} - /// `current`, or the fingerprint recorded with a `Fingerprint` row whose definition fingerprints to `current`. -fn recorded_baseline( +pub(crate) fn recorded_baseline( py: Python<'_>, current: String, recorded: &Bound<'_, PyAny>, @@ -79,17 +47,7 @@ fn recorded_baseline( .map_err(compiler_error) } -/// The definition fingerprint of a function, or the planner input error naming the function. -pub(crate) fn function_definition_hash( - py: Python<'_>, - function_name: &str, - (fingerprint_sql, language, dialect): (&str, &str, Option<&str>), -) -> PyResult { - py.compiler_detach(|| definition_fingerprint(fingerprint_sql, language, dialect)) - .map_err(|error| fingerprint_error(py, &format!("Function '{function_name}'"), error)) -} - -fn fingerprint_error(py: Python<'_>, subject: &str, message: String) -> PyErr { +pub(crate) fn fingerprint_error(py: Python<'_>, subject: &str, message: String) -> PyErr { if is_compiler_panic(&message) { return compiler_error(message); } @@ -114,27 +72,3 @@ fn fingerprint_error(py: Python<'_>, subject: &str, message: String) -> PyErr { Err(error) => error, } } - -/// The graph key of a `CompiledObjectKey`-like value: its resource type text and name. -pub(crate) fn graph_key(key: &Bound<'_, PyAny>) -> PyResult { - Ok(GraphKey { - kind: native_field(key, "resource_type")?.str()?.extract()?, - name: native_field(key, "name")?.extract()?, - }) -} - -/// Fingerprint a model query for change detection, naming the model when it cannot. -#[pyfunction] -#[pyo3(signature = (*, model_name, query_sql, dialect))] -pub(crate) fn model_definition_hash( - py: Python<'_>, - model_name: &str, - query_sql: &str, - dialect: Option<&str>, -) -> PyResult { - model_query_hash(py, model_name, (query_sql, dialect)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(model_definition_hash, module)?) -} diff --git a/crates/sqlbuild-python-project/src/planner_identity/_helpers/mod.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/mod.rs new file mode 100644 index 0000000000..35817cbc58 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod graph; +pub(crate) mod hashing; diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/baselined_function_definition_hash.rs b/crates/sqlbuild-python-project/src/planner_identity/main/baselined_function_definition_hash.rs new file mode 100644 index 0000000000..a64b87a9bd --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/baselined_function_definition_hash.rs @@ -0,0 +1,20 @@ +//! The function's definition fingerprint, or its recorded one while the recorded definition still fingerprints the same. + +use crate::planner_identity::_helpers::hashing::recorded_baseline; +use crate::planner_identity::main::function_definition_hash::function_definition_hash; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_analysis::sql_tokens::main::definition_fingerprint::definition_fingerprint; + +/// The function's definition fingerprint, or its recorded one while the recorded definition still fingerprints the same. +pub fn baselined_function_definition_hash( + py: Python<'_>, + function_name: &str, + (fingerprint_sql, language, dialect): (&str, &str, Option<&str>), + recorded: &Bound<'_, PyAny>, +) -> PyResult { + let current: String = + function_definition_hash(py, function_name, (fingerprint_sql, language, dialect))?; + recorded_baseline(py, current, recorded, |sql| { + definition_fingerprint(sql, language, dialect) + }) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/baselined_model_query_hash.rs b/crates/sqlbuild-python-project/src/planner_identity/main/baselined_model_query_hash.rs new file mode 100644 index 0000000000..4327a91cd3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/baselined_model_query_hash.rs @@ -0,0 +1,19 @@ +//! The model's query fingerprint, or its recorded one while the recorded SQL still fingerprints the same. + +use crate::planner_identity::_helpers::hashing::model_query_hash; +use crate::planner_identity::_helpers::hashing::recorded_baseline; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_analysis::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; + +/// The model's query fingerprint, or its recorded one while the recorded SQL still fingerprints the same. +pub fn baselined_model_query_hash( + py: Python<'_>, + model_name: &str, + (query_sql, dialect): (&str, Option<&str>), + recorded: &Bound<'_, PyAny>, +) -> PyResult { + let current: String = model_query_hash(py, model_name, (query_sql, dialect))?; + recorded_baseline(py, current, recorded, |sql| { + compiler_query_fingerprint(sql, dialect) + }) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps.rs b/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps.rs new file mode 100644 index 0000000000..4e7fb11f96 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps.rs @@ -0,0 +1,54 @@ +//! Execution dependencies kept native until a public graph or plan asks for its dict view. + +use crate::compiled_views::main::project::compiled_models::compiled_models; +use crate::compiled_views::main::project::functions::functions; +use crate::compiled_views::main::project::seeds::seeds; +use crate::compiled_views::main::project::sources::sources; +use crate::compiled_views::main::project::sql_tests::sql_tests; +use crate::graph::main::nodes::native_key::native_key; +use crate::planner_identity::_helpers::graph::TABLE_FN_TEST_MODE; +use crate::references::main::native_execution_gate_edges::native_execution_gate_edges; +use crate::references::models::{GateRow, HookRow}; +use crate::resource_views::main::resource_reads::graph_row::graph_row; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::planner_identity::main::execution_upstream::execution_upstream; +use sqlbuild_analysis::planner_identity::models::ExecutionTest; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Execution dependencies kept native until a public graph or plan asks for its dict view. +pub fn execution_deps(project: &Bound<'_, PyAny>) -> PyResult)>> { + let mut resources: Vec<(ObjectKey, Vec)> = Vec::new(); + let collections: [Bound<'_, PyAny>; 4] = [ + compiled_models(project)?, + sources(project)?, + seeds(project)?, + functions(project)?, + ]; + for collection in collections { + for resource in collection.try_iter()? { + let row = graph_row(&resource?)?; + resources.push((row.key, row.deps)); + } + } + let mut tests: Vec = Vec::new(); + for test in sql_tests(project)?.try_iter()? { + let test = test?; + tests.push(ExecutionTest { + key: native_key(&native_field(&test, "key")?)?, + scope_deps: native_field(&test, "scope_deps")? + .try_iter()? + .map(|key| native_key(&key?)) + .collect::>()?, + table_function: native_field(&test, "mode")?.str()?.extract::()? + == TABLE_FN_TEST_MODE, + }); + } + let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; + let injected: Vec<_> = gates + .into_iter() + .map(|gate| (gate.gated, gate.read)) + .chain(hooks.into_iter().map(|hook| (hook.gated, hook.read))) + .collect(); + Ok(execution_upstream(resources, &tests, &injected)) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/execution_order.rs b/crates/sqlbuild-python-project/src/planner_identity/main/execution_order.rs new file mode 100644 index 0000000000..d0bb0282e4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/execution_order.rs @@ -0,0 +1,33 @@ +//! Stable native execution order, projecting no Python graph or key objects. + +use crate::planner_identity::_helpers::graph::planner_input_error; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::planner_identity::main::topological_order::topological_order; +use sqlbuild_analysis::planner_identity::models::GraphKey; +use std::collections::HashMap; + +/// Stable native execution order, projecting no Python graph or key objects. +pub fn execution_order( + py: Python<'_>, + upstream: &[(ObjectKey, Vec)], + origins: &[((ObjectKey, ObjectKey), String)], +) -> PyResult> { + let key = |(kind, name): &ObjectKey| GraphKey { + kind: kind.clone(), + name: name.clone(), + }; + let edges: Vec<_> = upstream + .iter() + .map(|(node, deps)| (key(node), deps.iter().map(key).collect())) + .collect(); + let origins: HashMap<(GraphKey, GraphKey), String> = origins + .iter() + .map(|((gated, read), origin)| ((key(gated), key(read)), origin.clone())) + .collect(); + Ok(topological_order(&edges, &origins) + .map_err(|message| planner_input_error(py, message))? + .into_iter() + .map(|key| (key.kind, key.name)) + .collect()) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins.rs b/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins.rs new file mode 100644 index 0000000000..75e0769e49 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins.rs @@ -0,0 +1,78 @@ +//! Injected-edge descriptions remain native for cycle and buildability diagnostics. + +use crate::compiled_views::main::project::sql_tests::sql_tests; +use crate::graph::main::nodes::native_key::native_key; +use crate::graph::models::NativeProjectGraph; +use crate::planner_identity::_helpers::graph::hook_label; +use crate::references::main::native_execution_gate_edges::native_execution_gate_edges; +use crate::references::models::{GateRow, HookRow}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::main::edge_keys::edge_keys; +use sqlbuild_analysis::graph::models::EdgeDirection; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::HashSet; + +/// Injected-edge descriptions remain native for cycle and buildability diagnostics. +pub fn execution_origins( + project: &Bound<'_, PyAny>, +) -> PyResult> { + let mut origins: Vec<((ObjectKey, ObjectKey), String)> = Vec::new(); + for test in sql_tests(project)?.try_iter()? { + let test = test?; + let test_name: String = native_field(&test, "name")?.extract()?; + for target_key in native_field(&test, "scope_deps")?.try_iter()? { + let target_key = native_key(&target_key?)?; + let target_name = &target_key.1; + let description = format!("SQL test '{test_name}' runs before '{target_name}'"); + origins.push(( + (target_key, native_key(&native_field(&test, "key")?)?), + description, + )); + } + } + let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; + let lineage = native_field(project, "lineage_graph")?; + let lineage = lineage.cast::()?.borrow(); + let is_lineage = |gated: &ObjectKey, read: &ObjectKey| { + edge_keys(lineage.graph(), gated, EdgeDirection::Upstream).contains(read) + }; + let mut seen: HashSet<_> = origins.iter().map(|(edge, _)| edge.clone()).collect(); + for GateRow { + audit_name, + target, + gated, + read, + } in &gates + { + if is_lineage(gated, read) || !seen.insert((gated.clone(), read.clone())) { + continue; + } + origins.push(( + (gated.clone(), read.clone()), + format!("audit '{audit_name}' on '{}' reads '{}'", target.1, read.1), + )); + } + for HookRow { + hook_name, + gated, + read, + hook_type: kind, + .. + } in &hooks + { + if is_lineage(gated, read) || !seen.insert((gated.clone(), read.clone())) { + continue; + } + origins.push(( + (gated.clone(), read.clone()), + format!( + "{} on '{}' reads '{}'", + hook_label(hook_name, kind), + gated.1, + read.1 + ), + )); + } + Ok(origins) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/function_definition_hash.rs b/crates/sqlbuild-python-project/src/planner_identity/main/function_definition_hash.rs new file mode 100644 index 0000000000..4deefe82b7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/function_definition_hash.rs @@ -0,0 +1,16 @@ +//! The definition fingerprint of a function, or the planner input error naming the function. + +use crate::planner_identity::_helpers::hashing::fingerprint_error; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::sql_tokens::main::definition_fingerprint::definition_fingerprint; +use sqlbuild_python_base::types::CompilerDetach; + +/// The definition fingerprint of a function, or the planner input error naming the function. +pub fn function_definition_hash( + py: Python<'_>, + function_name: &str, + (fingerprint_sql, language, dialect): (&str, &str, Option<&str>), +) -> PyResult { + py.compiler_detach(|| definition_fingerprint(fingerprint_sql, language, dialect)) + .map_err(|error| fingerprint_error(py, &format!("Function '{function_name}'"), error)) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs b/crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs new file mode 100644 index 0000000000..62d4be2fcb --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs @@ -0,0 +1,13 @@ +//! The graph key of a `CompiledObjectKey`-like value: its resource type text and name. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::planner_identity::models::GraphKey; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The graph key of a `CompiledObjectKey`-like value: its resource type text and name. +pub fn graph_key(key: &Bound<'_, PyAny>) -> PyResult { + Ok(GraphKey { + kind: native_field(key, "resource_type")?.str()?.extract()?, + name: native_field(key, "name")?.extract()?, + }) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/mod.rs b/crates/sqlbuild-python-project/src/planner_identity/main/mod.rs new file mode 100644 index 0000000000..d007f0bb8b --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/mod.rs @@ -0,0 +1,10 @@ +pub mod baselined_function_definition_hash; +pub mod baselined_model_query_hash; +pub mod execution_deps; +pub mod execution_order; +pub mod execution_origins; +pub mod function_definition_hash; +pub mod graph_key; +pub mod model_definition_hash; +pub mod register_graph; +pub mod register_hashing; diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/model_definition_hash.rs b/crates/sqlbuild-python-project/src/planner_identity/main/model_definition_hash.rs new file mode 100644 index 0000000000..22837bfd48 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/model_definition_hash.rs @@ -0,0 +1,17 @@ +//! Fingerprint a model query for change detection, naming the model when it cannot. + +use crate::planner_identity::_helpers::hashing::model_query_hash; +use pyo3::prelude::{PyResult, Python}; +use pyo3::pyfunction; + +/// Fingerprint a model query for change detection, naming the model when it cannot. +#[pyfunction] +#[pyo3(signature = (*, model_name, query_sql, dialect))] +pub fn model_definition_hash( + py: Python<'_>, + model_name: &str, + query_sql: &str, + dialect: Option<&str>, +) -> PyResult { + model_query_hash(py, model_name, (query_sql, dialect)) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs b/crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs new file mode 100644 index 0000000000..c7aa14798a --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs @@ -0,0 +1,25 @@ +//! Register the graph functions and classes. + +use crate::planner_identity::_helpers::graph::build_downstream_deps; +use crate::planner_identity::_helpers::graph::build_downstream_model_name_closure; +use crate::planner_identity::_helpers::graph::build_execution_edge_origins; +use crate::planner_identity::_helpers::graph::build_execution_upstream_deps; +use crate::planner_identity::_helpers::graph::check_buildability; +use crate::planner_identity::_helpers::graph::missing_upstream_message; +use crate::planner_identity::_helpers::graph::topologically_order_keys; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the graph functions and classes. +pub fn register_graph(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_execution_upstream_deps, module)?)?; + module.add_function(wrap_pyfunction!(build_execution_edge_origins, module)?)?; + module.add_function(wrap_pyfunction!(build_downstream_deps, module)?)?; + module.add_function(wrap_pyfunction!(topologically_order_keys, module)?)?; + module.add_function(wrap_pyfunction!( + build_downstream_model_name_closure, + module + )?)?; + module.add_function(wrap_pyfunction!(check_buildability, module)?)?; + module.add_function(wrap_pyfunction!(missing_upstream_message, module)?) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/register_hashing.rs b/crates/sqlbuild-python-project/src/planner_identity/main/register_hashing.rs new file mode 100644 index 0000000000..86e326bd6e --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/register_hashing.rs @@ -0,0 +1,10 @@ +//! Register the hashing functions and classes. + +use crate::planner_identity::main::model_definition_hash::model_definition_hash; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the hashing functions and classes. +pub fn register_hashing(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(model_definition_hash, module)?) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/mod.rs b/crates/sqlbuild-python-project/src/planner_identity/mod.rs new file mode 100644 index 0000000000..5e15399939 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/mod.rs @@ -0,0 +1,4 @@ +//! Planner identities: the graph and its hashes. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/by_name.rs b/crates/sqlbuild-python-project/src/planner_phases/main/by_name.rs new file mode 100644 index 0000000000..130b9c42fc --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/by_name.rs @@ -0,0 +1,21 @@ +//! The project nodes of `field` by name. + +use crate::compiled_views::main::project::collection::collection; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +/// The project nodes of `field` by name. +pub fn by_name<'py>( + project: &Bound<'py, PyAny>, + field: &str, +) -> PyResult> { + let nodes = pyo3::types::PyDict::new(project.py()); + for node in collection(project, field)?.try_iter()? { + let node = node?; + pyo3::types::PyDictMethods::set_item( + &nodes, + crate::resource_views::main::resource_reads::name::name(&node)?, + node, + )?; + } + Ok(nodes) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/config_str.rs b/crates/sqlbuild-python-project/src/planner_phases/main/config_str.rs new file mode 100644 index 0000000000..da19090d81 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/config_str.rs @@ -0,0 +1,10 @@ +//! A model config value as text, `None` when absent or not text. + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_front::model_config::main::config::config_text::config_text; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; + +/// A model config value as text, `None` when absent or not text. +pub fn config_str(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { + config_text(model.py(), &model_values(model)?, key) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/construct.rs b/crates/sqlbuild-python-project/src/planner_phases/main/construct.rs new file mode 100644 index 0000000000..daece070e4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/construct.rs @@ -0,0 +1,15 @@ +//! The staying-Python class `class(**fields)`. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_host::entries::models::HostId; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; + +/// The staying-Python class `class(**fields)`. +pub fn construct<'py>( + py: Python<'py>, + class: impl Into, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&host_entry(py, class)?, fields) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/frozenset.rs b/crates/sqlbuild-python-project/src/planner_phases/main/frozenset.rs new file mode 100644 index 0000000000..e59e3f13db --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/frozenset.rs @@ -0,0 +1,9 @@ +//! A frozenset of the items. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyFrozenSet; + +/// A frozenset of the items. +pub fn frozenset<'py>(py: Python<'py>, items: &[Bound<'py, PyAny>]) -> PyResult> { + Ok(PyFrozenSet::new(py, items)?.into_any()) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/member.rs b/crates/sqlbuild-python-project/src/planner_phases/main/member.rs new file mode 100644 index 0000000000..14dddccf85 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/member.rs @@ -0,0 +1,14 @@ +//! A staying-Python enum member from its value. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_host::entries::models::HostId; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; + +/// A staying-Python enum member from its value. +pub fn member<'py>( + py: Python<'py>, + class: impl Into, + value: &str, +) -> PyResult> { + host_entry(py, class)?.call1((value,)) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/mod.rs b/crates/sqlbuild-python-project/src/planner_phases/main/mod.rs new file mode 100644 index 0000000000..f8acd3faea --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/mod.rs @@ -0,0 +1,17 @@ +pub mod by_name; +pub mod config_str; +pub mod construct; +pub mod frozenset; +pub mod member; +pub mod model_key; +pub mod native; +pub mod native_member; +pub mod object; +pub mod optional_text; +pub mod planner_error; +pub mod planner_model; +pub mod replace; +pub mod sorted_names; +pub mod strings; +pub mod text; +pub mod tuple; diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/model_key.rs b/crates/sqlbuild-python-project/src/planner_phases/main/model_key.rs new file mode 100644 index 0000000000..c8c1e92d9b --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/model_key.rs @@ -0,0 +1,21 @@ +//! The compiled object key of the model `name`. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; + +/// The compiled object key of the model `name`. +pub fn model_key<'py>(py: Python<'py>, name: &Bound<'py, PyAny>) -> PyResult> { + new_record_with( + py, + RecordId::CompiledObjectKey, + &[ + ( + "resource_type", + enum_member(py, RecordId::CompiledResourceType, "MODEL")?, + ), + ("name", name.clone()), + ], + ) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/native.rs b/crates/sqlbuild-python-project/src/planner_phases/main/native.rs new file mode 100644 index 0000000000..c1c20a1760 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/native.rs @@ -0,0 +1,14 @@ +//! The native record `name(**fields)`. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; + +/// The native record `name(**fields)`. +pub fn native<'py>( + py: Python<'py>, + name: RecordId, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + new_record_with(py, name, fields) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/native_member.rs b/crates/sqlbuild-python-project/src/planner_phases/main/native_member.rs new file mode 100644 index 0000000000..a1112e6bd3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/native_member.rs @@ -0,0 +1,14 @@ +//! A native string enum member from its value. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::new_record::new_record; +use sqlbuild_python_base::records::types::RecordId; + +/// A native string enum member from its value. +pub fn native_member<'py>( + py: Python<'py>, + class: RecordId, + value: &str, +) -> PyResult> { + new_record(py, class, (value,), None) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/object.rs b/crates/sqlbuild-python-project/src/planner_phases/main/object.rs new file mode 100644 index 0000000000..683570850d --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/object.rs @@ -0,0 +1,13 @@ +//! A native helper's Rust result as a Python object. + +use pyo3::prelude::{Bound, PyAny, PyErr, PyResult, Python}; +use pyo3::{BoundObject, IntoPyObject}; + +/// A native helper's Rust result as a Python object. +pub fn object<'py, T>(py: Python<'py>, value: T) -> PyResult> +where + T: IntoPyObject<'py>, + PyErr: From, +{ + Ok(value.into_pyobject(py)?.into_bound().into_any()) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/optional_text.rs b/crates/sqlbuild-python-project/src/planner_phases/main/optional_text.rs new file mode 100644 index 0000000000..55bb892586 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/optional_text.rs @@ -0,0 +1,9 @@ +//! `None`, or the text. + +use crate::planner_phases::main::text::text; +use pyo3::prelude::{Bound, PyAny, Python}; + +/// `None`, or the text. +pub fn optional_text<'py>(py: Python<'py>, value: Option<&str>) -> Bound<'py, PyAny> { + value.map_or_else(|| py.None().into_bound(py), |value| text(py, value)) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/planner_error.rs b/crates/sqlbuild-python-project/src/planner_phases/main/planner_error.rs new file mode 100644 index 0000000000..0e363eefd3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/planner_error.rs @@ -0,0 +1,29 @@ +//! `PlannerInputError(message, code=code, help=help)`. + +use crate::planner_phases::main::optional_text::optional_text; +use pyo3::prelude::{PyAnyMethods, PyErr, Python}; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; + +/// `PlannerInputError(message, code=code, help=help)`. +pub fn planner_error( + py: Python<'_>, + message: &str, + (code, help): (Option<&str>, Option<&str>), +) -> PyErr { + let raised = error_class(py, ErrorId::PlannerInput).and_then(|class| { + let fields = keywords( + py, + &[ + ("code", optional_text(py, code)), + ("help", optional_text(py, help)), + ], + )?; + class.call((message,), Some(&fields)) + }); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/planner_model.rs b/crates/sqlbuild-python-project/src/planner_phases/main/planner_model.rs new file mode 100644 index 0000000000..299978c4b4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/planner_model.rs @@ -0,0 +1,14 @@ +//! A planner model dataclass. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; + +/// A planner model dataclass. +pub fn planner_model<'py>( + py: Python<'py>, + name: RecordId, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + new_record_with(py, name, fields) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/replace.rs b/crates/sqlbuild-python-project/src/planner_phases/main/replace.rs new file mode 100644 index 0000000000..735c470993 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/replace.rs @@ -0,0 +1,13 @@ +//! `dataclasses.replace(value, **fields)`. + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; + +/// `dataclasses.replace(value, **fields)`. +pub fn replace<'py>( + value: &Bound<'py, PyAny>, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + replaced_keywords(value, &keywords(value.py(), fields)?) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs b/crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs new file mode 100644 index 0000000000..93a2a16f3d --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs @@ -0,0 +1,11 @@ +//! `sorted(items)` as a tuple of strings. + +use crate::planner_phases::main::text::text; +use crate::planner_phases::main::tuple::tuple; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; + +/// `sorted(items)` as a tuple of strings. +pub fn sorted_names<'py>(py: Python<'py>, mut names: Vec) -> PyResult> { + names.sort(); + tuple(py, names.iter().map(|name| text(py, name)).collect()) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/strings.rs b/crates/sqlbuild-python-project/src/planner_phases/main/strings.rs new file mode 100644 index 0000000000..21950fbc5b --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/strings.rs @@ -0,0 +1,11 @@ +//! The strings of any Python iterable, in iteration order. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +/// The strings of any Python iterable, in iteration order. +pub fn strings(values: &Bound<'_, PyAny>) -> PyResult> { + values + .try_iter()? + .map(|value| value?.extract::()) + .collect() +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/text.rs b/crates/sqlbuild-python-project/src/planner_phases/main/text.rs new file mode 100644 index 0000000000..b7116c021f --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/text.rs @@ -0,0 +1,9 @@ +//! A Python `str`. + +use pyo3::prelude::{Bound, PyAny, Python}; +use pyo3::types::PyString; + +/// A Python `str`. +pub fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/main/tuple.rs b/crates/sqlbuild-python-project/src/planner_phases/main/tuple.rs new file mode 100644 index 0000000000..c482382182 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/main/tuple.rs @@ -0,0 +1,9 @@ +//! A tuple of the items. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyTuple; + +/// A tuple of the items. +pub fn tuple<'py>(py: Python<'py>, items: Vec>) -> PyResult> { + Ok(PyTuple::new(py, items)?.into_any()) +} diff --git a/crates/sqlbuild-python-project/src/planner_phases/mod.rs b/crates/sqlbuild-python-project/src/planner_phases/mod.rs new file mode 100644 index 0000000000..74d7b06897 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_phases/mod.rs @@ -0,0 +1,3 @@ +//! Planner phase objects and scopes. + +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/loaders.rs similarity index 74% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs rename to crates/sqlbuild-python-project/src/planner_scope/_helpers/loaders.rs index 4d9bde263f..bce0b2f527 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/loaders.rs @@ -1,12 +1,12 @@ //! Loader dependency mapping and synthetic source boundary construction. -use crate::bindings::_helpers::graph::keys::KeyCache; -use crate::bindings::_helpers::graph::views::edge_dict; -use crate::bindings::_helpers::planner_scope::values::{ - dependency_names, edges, keys, loader_graph, loader_names, project_graph, -}; +use crate::graph::main::project::edge_dict::edge_dict; +use crate::graph::models::KeyCache; +use crate::planner_scope::main::edges::edges; +use crate::planner_scope::main::keys::keys; +use crate::planner_scope::main::project_graph::project_graph; use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, pyfunction}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::planner_scope::main::{ expand_loaders::expand_loaders, intermediate_sources::intermediate_sources, @@ -17,18 +17,6 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::loader_parts::loader_parts; use std::collections::HashSet; -#[pyfunction] -#[pyo3(signature = (*, loader_function, loader_functions))] -pub(crate) fn upstream_loader_dependency_names<'py>( - loader_function: &Bound<'py, PyAny>, - loader_functions: &Bound<'py, PyAny>, -) -> PyResult> { - let graph = loader_graph(loader_functions)?; - let names = loader_names(loader_functions)?; - let deps = dependency_names(loader_function, &names)?; - PyTuple::new(loader_function.py(), graph.upstream(&deps)) -} - pub(crate) fn loader_to_source_entry<'py>( project: &Bound<'py, PyAny>, loader: &Bound<'py, PyAny>, @@ -91,22 +79,6 @@ pub(crate) fn intermediate_map<'py>( Ok(result) } -#[pyfunction] -#[pyo3(signature = (*, project, selected_keys))] -pub(crate) fn build_intermediate_source_map<'py>( - project: &Bound<'py, PyAny>, - selected_keys: &Bound<'py, PyAny>, -) -> PyResult> { - intermediate_map(project, &keys(selected_keys)?, false) -} - -pub(crate) fn build_upstream_intermediate_source_map<'py>( - project: &Bound<'py, PyAny>, - selected_keys: &Bound<'py, PyAny>, -) -> PyResult> { - intermediate_map(project, &keys(selected_keys)?, true) -} - #[pyfunction] #[pyo3(signature = (*, project, selected_keys, upstream_deps, executable_dependency_source_keys = None))] pub(crate) fn expand_selected_loader_dependencies<'py>( diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/mod.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/mod.rs new file mode 100644 index 0000000000..df51068857 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod loaders; +pub(crate) mod retained; +pub(crate) mod scope; +pub(crate) mod sources; +pub(crate) mod values; diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs new file mode 100644 index 0000000000..bd56463a03 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/retained.rs @@ -0,0 +1,56 @@ +//! Native planner scope ownership, with projection only at public record boundaries. + +use crate::planner_scope::main::edges::edges; +use crate::planner_scope::main::keys::keys; +use crate::planner_scope::models::RetainedScope; +use crate::planner_scope::models::ScopeData; +use std::collections::HashMap; +use std::sync::Arc; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +use crate::native_host::models::ModelHandle; +use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; + +/// Read native ownership directly, or import a caller-authored public scope at the API edge. +pub(crate) fn scope_data(value: &Bound<'_, PyAny>) -> PyResult> { + if let Ok(scope) = value.cast::() { + return Ok(scope.get().data.clone()); + } + let mut models: HashMap = HashMap::new(); + let mut model_order: Vec = Vec::new(); + for item in mapping_items(&field(value, "models_by_name")?)?.try_iter()? { + let item = item?; + let name: String = item.get_item(0)?.extract()?; + model_order.push(name.clone()); + models.insert(name, ModelHandle::read(&item.get_item(1)?)); + } + Ok(Arc::new(ScopeData { + upstream: edges(&field(value, "upstream_deps")?)?, + downstream: edges(&field(value, "downstream_deps")?)?, + names: mapping_items(&field(value, "all_keys")?)? + .try_iter()? + .map(|item| { + let item = item?; + Ok(( + item.get_item(0)?.extract()?, + crate::graph::main::nodes::native_key::native_key(&item.get_item(1)?)?, + )) + }) + .collect::>()?, + models, + model_order, + selected: keys(&field(value, "selected_keys")?)?.into_iter().collect(), + order: keys(&field(value, "execution_order")?)?, + user_selected: keys(&field(value, "user_selected_keys")?)? + .into_iter() + .collect(), + python_reads: field(value, "python_read_source_names")?.extract()?, + tests: field(value, "sql_test_selection")?.extract()?, + load_sources: field(value, "load_sources")?.is_truthy()?, + end_audit_sources: keys(&field(value, "end_audit_source_keys")?)? + .into_iter() + .collect(), + })) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs similarity index 78% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs rename to crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs index 3926e1fe22..639297c7ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs @@ -1,17 +1,20 @@ //! Planner scope assembly using native graph and selector operations. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::native_host::model_lookup::ModelHandle; -use crate::bindings::_helpers::planner_identity::graph::{ - execution_deps, execution_order, execution_origins, -}; -use crate::bindings::_helpers::planner_scope::retained::ScopeData; -use crate::bindings::_helpers::planner_scope::values::{keys, project_graph}; -use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; -use crate::bindings::_helpers::planner_selection::selectors::resolve_graph_keys; +use crate::compiled_views::main::compiled_models::name_object::name_object; +use crate::compiled_views::main::project::compiled_models::compiled_models; +use crate::compiled_views::main::project::sources::sources; +use crate::graph::models::NativeProjectGraph; +use crate::native_host::models::ModelHandle; +use crate::planner_identity::main::execution_deps::execution_deps; +use crate::planner_identity::main::execution_order::execution_order; +use crate::planner_identity::main::execution_origins::execution_origins; +use crate::planner_scope::main::keys::keys; +use crate::planner_scope::main::project_graph::project_graph; +use crate::planner_scope::models::ScopeData; +use crate::planner_selection::main::py_strings::py_strings; +use crate::planner_selection::main::resolve_graph_keys::resolve_graph_keys; +use crate::planner_selection::models::Carrier; +use crate::resource_views::main::source_reads::source_entry::source_entry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; use sqlbuild_analysis::assembly::project::types::ObjectKey; @@ -78,10 +81,12 @@ pub(crate) fn build_planner_scope<'py>( }; if auto_load { let mut managed: HashSet = HashSet::new(); - for source in project_reads::sources(&project)?.try_iter()? { + for source in sources(&project)?.try_iter()? { let source = source?; - if !native_field(&source_reads::source_entry(&source)?, "loader")?.is_none() { - managed.insert(source_reads::name(&source)?.extract()?); + if !native_field(&source_entry(&source)?, "loader")?.is_none() { + managed.insert( + crate::resource_views::main::source_reads::name::name(&source)?.extract()?, + ); } } let loaded = managed_sources(&selected, &upstream, &managed); @@ -110,9 +115,9 @@ pub(crate) fn build_planner_scope<'py>( let mut models: std::collections::HashMap = std::collections::HashMap::new(); let mut model_order: Vec = Vec::new(); - for model in project_reads::models(&project)?.try_iter()? { + for model in compiled_models(&project)?.try_iter()? { let model = model?; - let name: String = model_reads::name_object(&model)?.extract()?; + let name: String = name_object(&model)?.extract()?; if !models.contains_key(&name) { model_order.push(name.clone()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs similarity index 70% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs rename to crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs index f603c490ee..36585d9e3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs @@ -1,17 +1,13 @@ //! Existing source and load plan models materialised at the Python boundary. +use crate::planner_scope::main::edges::edges; +use crate::planner_scope::main::keys::keys; +use crate::planner_scope::main::source_load_map::source_load_map; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::graph::keys::native_key; -use crate::bindings::_helpers::planner_scope::loaders::intermediate_map; -use crate::bindings::_helpers::planner_scope::values::{edges, keys}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; use pyo3::types::PyDictMethods; use pyo3::types::{PyBool, PyDict, PyTuple}; -use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; use sqlbuild_analysis::planner_scope::constants::SOURCE; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -32,32 +28,6 @@ pub(crate) fn build_source_load_map<'py>( ) } -/// Select source entries from native reachability; the resulting entries cross the executor edge. -pub(crate) fn source_load_map<'py>( - project: &Bound<'py, PyAny>, - selected: &[ObjectKey], - upstream_deps: Option<&[(ObjectKey, Vec)]>, -) -> PyResult> { - let relevant = upstream_deps.map(|edges| reachable_keys(selected, edges)); - let result = PyDict::new(project.py()); - for source in project_reads::sources(project)?.try_iter()? { - let source = source?; - if let Some(keys) = &relevant - && !keys.contains(&native_key(&source_reads::key(&source)?)?) - { - continue; - } - let entry = source_reads::source_entry(&source)?; - result.set_item(native_field(&entry, "name")?, entry)?; - } - for upstream in [false, true] { - for (key, entry) in intermediate_map(project, selected, upstream)?.iter() { - result.set_item(key, entry)?; - } - } - Ok(result) -} - #[pyfunction] #[pyo3(signature = (*, execution_order, selected_keys, source_map, is_reload))] pub(crate) fn build_source_load_entries<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs similarity index 55% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs rename to crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs index 713ae4ab47..07251d4cdf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs @@ -1,32 +1,10 @@ //! Python function identity and compiled graph conversion. -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::graph::keys::native_key; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyDict; use pyo3::types::PyDictMethods; -use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::planner_scope::models::LoaderGraph; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; - -pub(crate) fn keys(values: &Bound<'_, PyAny>) -> PyResult> { - values - .try_iter()? - .map(|value| native_key(&value?)) - .collect() -} - -pub(crate) fn edges(values: &Bound<'_, PyAny>) -> PyResult)>> { - mapping_items(values)? - .try_iter()? - .map(|item| { - let (key, deps): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; - Ok((native_key(&key)?, keys(&deps)?)) - }) - .collect() -} /// Resolve dependencies using the same Python dictionary lookup as the authored function boundary. pub(crate) fn loader_names<'py>(loaders: &Bound<'py, PyAny>) -> PyResult> { @@ -66,15 +44,3 @@ pub(crate) fn loader_graph(loaders: &Bound<'_, PyAny>) -> PyResult } Ok(graph) } - -pub(crate) fn project_graph(project: &Bound<'_, PyAny>) -> PyResult { - let mut graph = loader_graph(&native_field(project, "loader_functions")?)?; - for source in project_reads::sources(project)?.try_iter()? { - let entry = source_reads::source_entry(&source?)?; - graph.sources.push(( - native_field(&entry, "name")?.extract()?, - native_field(&entry, "loader")?.extract()?, - )); - } - Ok(graph) -} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map.rs new file mode 100644 index 0000000000..307f646797 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map.rs @@ -0,0 +1,15 @@ +//! Build intermediate source map. + +use crate::planner_scope::_helpers::loaders::intermediate_map; +use crate::planner_scope::main::keys::keys; +use pyo3::prelude::{Bound, PyAny, PyResult, pyfunction}; +use pyo3::types::PyDict; + +#[pyfunction] +#[pyo3(signature = (*, project, selected_keys))] +pub fn build_intermediate_source_map<'py>( + project: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult> { + intermediate_map(project, &keys(selected_keys)?, false) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map_function.rs new file mode 100644 index 0000000000..56dc7ac73f --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_intermediate_source_map_function.rs @@ -0,0 +1,10 @@ +//! `build_intermediate_source_map` as a function of `module`. + +use crate::planner_scope::main::build_intermediate_source_map::build_intermediate_source_map; + +/// `build_intermediate_source_map` as a function of `module`. +pub fn build_intermediate_source_map_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_intermediate_source_map, module) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_planner_scope.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_planner_scope.rs new file mode 100644 index 0000000000..c474c1deb4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_planner_scope.rs @@ -0,0 +1,14 @@ +//! Build planner scope. + +use crate::planner_scope::_helpers::scope; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyDict; + +/// Build planner scope. +#[inline] +pub fn build_planner_scope<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + scope::build_planner_scope(py, options) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries.rs new file mode 100644 index 0000000000..e7b23b78c3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries.rs @@ -0,0 +1,16 @@ +//! Build source load entries. + +use crate::planner_scope::_helpers::sources; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::{PyDict, PyTuple}; + +/// Build source load entries. +#[inline] +pub fn build_source_load_entries<'py>( + execution_order: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, + source_map: &Bound<'py, PyDict>, + is_reload: bool, +) -> PyResult> { + sources::build_source_load_entries(execution_order, selected_keys, source_map, is_reload) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries_function.rs new file mode 100644 index 0000000000..20292d5327 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_entries_function.rs @@ -0,0 +1,10 @@ +//! `build_source_load_entries` as a function of `module`. + +use crate::planner_scope::_helpers::sources::build_source_load_entries; + +/// `build_source_load_entries` as a function of `module`. +pub fn build_source_load_entries_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_source_load_entries, module) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_map_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_map_function.rs new file mode 100644 index 0000000000..b5ae8b9a02 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_source_load_map_function.rs @@ -0,0 +1,10 @@ +//! `build_source_load_map` as a function of `module`. + +use crate::planner_scope::_helpers::sources::build_source_load_map; + +/// `build_source_load_map` as a function of `module`. +pub fn build_source_load_map_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(build_source_load_map, module) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/build_upstream_intermediate_source_map.rs b/crates/sqlbuild-python-project/src/planner_scope/main/build_upstream_intermediate_source_map.rs new file mode 100644 index 0000000000..7ddd568f53 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/build_upstream_intermediate_source_map.rs @@ -0,0 +1,14 @@ +//! Build upstream intermediate source map. + +use crate::planner_scope::_helpers::loaders::intermediate_map; +use crate::planner_scope::main::keys::keys; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyDict; + +/// Build upstream intermediate source map. +pub fn build_upstream_intermediate_source_map<'py>( + project: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult> { + intermediate_map(project, &keys(selected_keys)?, true) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/edges.rs b/crates/sqlbuild-python-project/src/planner_scope/main/edges.rs new file mode 100644 index 0000000000..9b5e76e884 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/edges.rs @@ -0,0 +1,18 @@ +//! Edges. + +use crate::graph::main::nodes::native_key::native_key; +use crate::planner_scope::main::keys::keys; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; + +/// Edges. +pub fn edges(values: &Bound<'_, PyAny>) -> PyResult)>> { + mapping_items(values)? + .try_iter()? + .map(|item| { + let (key, deps): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + Ok((native_key(&key)?, keys(&deps)?)) + }) + .collect() +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/expand_selected_loader_dependencies_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/expand_selected_loader_dependencies_function.rs new file mode 100644 index 0000000000..5748044e8d --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/expand_selected_loader_dependencies_function.rs @@ -0,0 +1,10 @@ +//! `expand_selected_loader_dependencies` as a function of `module`. + +use crate::planner_scope::_helpers::loaders::expand_selected_loader_dependencies; + +/// `expand_selected_loader_dependencies` as a function of `module`. +pub fn expand_selected_loader_dependencies_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(expand_selected_loader_dependencies, module) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/keys.rs b/crates/sqlbuild-python-project/src/planner_scope/main/keys.rs new file mode 100644 index 0000000000..5d86ce0c14 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/keys.rs @@ -0,0 +1,13 @@ +//! Keys. + +use crate::graph::main::nodes::native_key::native_key; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; + +/// Keys. +pub fn keys(values: &Bound<'_, PyAny>) -> PyResult> { + values + .try_iter()? + .map(|value| native_key(&value?)) + .collect() +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/mod.rs b/crates/sqlbuild-python-project/src/planner_scope/main/mod.rs new file mode 100644 index 0000000000..03a88c4a60 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/mod.rs @@ -0,0 +1,16 @@ +pub mod build_intermediate_source_map; +pub mod build_intermediate_source_map_function; +pub mod build_planner_scope; +pub mod build_source_load_entries; +pub mod build_source_load_entries_function; +pub mod build_source_load_map_function; +pub mod build_upstream_intermediate_source_map; +pub mod edges; +pub mod expand_selected_loader_dependencies_function; +pub mod keys; +pub mod project_graph; +pub mod project_scope; +pub mod scope_data; +pub mod source_load_map; +pub mod upstream_loader_dependency_names; +pub mod upstream_loader_dependency_names_function; diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/project_graph.rs b/crates/sqlbuild-python-project/src/planner_scope/main/project_graph.rs new file mode 100644 index 0000000000..23304c0abc --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/project_graph.rs @@ -0,0 +1,21 @@ +//! Project graph. + +use crate::compiled_views::main::project::sources::sources; +use crate::planner_scope::_helpers::values::loader_graph; +use crate::resource_views::main::source_reads::source_entry::source_entry; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::planner_scope::models::LoaderGraph; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Project graph. +pub fn project_graph(project: &Bound<'_, PyAny>) -> PyResult { + let mut graph = loader_graph(&native_field(project, "loader_functions")?)?; + for source in sources(project)?.try_iter()? { + let entry = source_entry(&source?)?; + graph.sources.push(( + native_field(&entry, "name")?.extract()?, + native_field(&entry, "loader")?.extract()?, + )); + } + Ok(graph) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs b/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs new file mode 100644 index 0000000000..734d1789c4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/project_scope.rs @@ -0,0 +1,42 @@ +//! Preserve public record identity and dataclass behavior on explicit projection. + +use crate::planner_scope::models::RetainedScope; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::models::NativeRecord; +use sqlbuild_python_base::records::types::RecordId; + +/// Preserve public record identity and dataclass behavior on explicit projection. +pub fn project_scope<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let Ok(scope) = value.cast::() else { + return Ok(value.clone()); + }; + scope + .get() + .projection + .get_or_try_init(value.py(), || { + let names = [ + "upstream_deps", + "downstream_deps", + "all_keys", + "models_by_name", + "selected_keys", + "execution_order", + "user_selected_keys", + "python_read_source_names", + "sql_test_selection", + "load_sources", + "end_audit_source_keys", + ]; + let fields = names + .iter() + .map(|name| Ok((*name, scope.get().data.field(value.py(), name)?))) + .collect::>>()?; + Ok::<_, pyo3::PyErr>( + new_record_with(value.py(), RecordId::PlannerScope, &fields)? + .cast_into::() + .map(Bound::unbind)?, + ) + }) + .map(|value| value.bind(scope.py()).clone().into_any()) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/scope_data.rs b/crates/sqlbuild-python-project/src/planner_scope/main/scope_data.rs new file mode 100644 index 0000000000..bddaaf8095 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/scope_data.rs @@ -0,0 +1,12 @@ +//! Read native ownership directly, or import a caller-authored public scope at the API edge. + +use crate::planner_scope::_helpers::retained; +use crate::planner_scope::models::ScopeData; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use std::sync::Arc; + +/// Read native ownership directly, or import a caller-authored public scope at the API edge. +#[inline] +pub fn scope_data(value: &Bound<'_, PyAny>) -> PyResult> { + retained::scope_data(value) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/source_load_map.rs b/crates/sqlbuild-python-project/src/planner_scope/main/source_load_map.rs new file mode 100644 index 0000000000..2595423ad1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/source_load_map.rs @@ -0,0 +1,40 @@ +//! Select source entries from native reachability; the resulting entries cross the executor edge. + +use crate::compiled_views::main::project::sources::sources; +use crate::graph::main::nodes::native_key::native_key; +use crate::planner_scope::_helpers::loaders::intermediate_map; +use crate::resource_views::main::source_reads::source_entry::source_entry; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Select source entries from native reachability; the resulting entries cross the executor edge. +pub fn source_load_map<'py>( + project: &Bound<'py, PyAny>, + selected: &[ObjectKey], + upstream_deps: Option<&[(ObjectKey, Vec)]>, +) -> PyResult> { + let relevant = upstream_deps.map(|edges| reachable_keys(selected, edges)); + let result = PyDict::new(project.py()); + for source in sources(project)?.try_iter()? { + let source = source?; + if let Some(keys) = &relevant + && !keys.contains(&native_key( + &crate::resource_views::main::source_reads::key::key(&source)?, + )?) + { + continue; + } + let entry = source_entry(&source)?; + result.set_item(native_field(&entry, "name")?, entry)?; + } + for upstream in [false, true] { + for (key, entry) in intermediate_map(project, selected, upstream)?.iter() { + result.set_item(key, entry)?; + } + } + Ok(result) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names.rs b/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names.rs new file mode 100644 index 0000000000..b65801d5ca --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names.rs @@ -0,0 +1,17 @@ +//! Upstream loader dependency names. + +use crate::planner_scope::_helpers::values::{dependency_names, loader_graph, loader_names}; +use pyo3::prelude::{Bound, PyAny, PyResult, pyfunction}; +use pyo3::types::PyTuple; + +#[pyfunction] +#[pyo3(signature = (*, loader_function, loader_functions))] +pub fn upstream_loader_dependency_names<'py>( + loader_function: &Bound<'py, PyAny>, + loader_functions: &Bound<'py, PyAny>, +) -> PyResult> { + let graph = loader_graph(loader_functions)?; + let names = loader_names(loader_functions)?; + let deps = dependency_names(loader_function, &names)?; + PyTuple::new(loader_function.py(), graph.upstream(&deps)) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names_function.rs b/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names_function.rs new file mode 100644 index 0000000000..4fb4f65fea --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/main/upstream_loader_dependency_names_function.rs @@ -0,0 +1,10 @@ +//! `upstream_loader_dependency_names` as a function of `module`. + +use crate::planner_scope::main::upstream_loader_dependency_names::upstream_loader_dependency_names; + +/// `upstream_loader_dependency_names` as a function of `module`. +pub fn upstream_loader_dependency_names_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(upstream_loader_dependency_names, module) +} diff --git a/crates/sqlbuild-python-project/src/planner_scope/mod.rs b/crates/sqlbuild-python-project/src/planner_scope/mod.rs new file mode 100644 index 0000000000..6b362db2c5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/mod.rs @@ -0,0 +1,5 @@ +//! The planner scope: loaders, sources, retained values and scope values. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-project/src/planner_scope/models.rs b/crates/sqlbuild-python-project/src/planner_scope/models.rs new file mode 100644 index 0000000000..91a26242c2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_scope/models.rs @@ -0,0 +1,131 @@ +//! Public records of planner scope. + +use crate::graph::main::project::edge_dict::edge_dict; +use crate::graph::models::KeyCache; +use crate::native_host::models::ModelHandle; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyFrozenSet; +use pyo3::types::PyTuple; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_python_base::records::models::NativeRecord; +use std::collections::HashMap; +use std::collections::HashSet; +use std::sync::Arc; + +/// The graph and selection used throughout planning; model values remain compiled-resource views. +#[derive(Debug)] +pub struct ScopeData { + pub upstream: Vec<(ObjectKey, Vec)>, + pub downstream: Vec<(ObjectKey, Vec)>, + pub names: Vec<(String, ObjectKey)>, + pub models: HashMap, + pub model_order: Vec, + pub selected: HashSet, + pub order: Vec, + pub user_selected: HashSet, + pub python_reads: HashSet, + pub tests: Py, + pub load_sources: bool, + pub end_audit_sources: HashSet, +} + +/// Internal carrier, never returned in place of a supported public PlannerScope record. +#[pyclass(frozen)] +pub struct RetainedScope { + pub data: Arc, + pub(crate) projection: PyOnceLock>, +} + +impl std::fmt::Debug for RetainedScope { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RetainedScope") + .finish_non_exhaustive() + } +} + +impl ScopeData { + pub fn object(self, py: Python<'_>) -> PyResult> { + Ok(Bound::new( + py, + RetainedScope { + data: Arc::new(self), + projection: PyOnceLock::new(), + }, + )? + .into_any()) + } + + pub fn model<'py>(&self, py: Python<'py>, name: &str) -> Option> { + self.models.get(name).map(|model| model.bind(py)) + } + + pub fn with_selected(&self, py: Python<'_>, selected: HashSet) -> Self { + Self { + upstream: self.upstream.clone(), + downstream: self.downstream.clone(), + names: self.names.clone(), + model_order: self.model_order.clone(), + models: self + .models + .iter() + .map(|(name, model)| (name.clone(), model.clone_ref(py))) + .collect(), + selected, + order: self.order.clone(), + user_selected: self.user_selected.clone(), + python_reads: self.python_reads.clone(), + tests: self.tests.clone_ref(py), + load_sources: self.load_sources, + end_audit_sources: self.end_audit_sources.clone(), + } + } + + /// Public field projection; internal graph consumers read the native fields above instead. + pub fn field<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + let cache = KeyCache::new(py)?; + let key_set = |keys: &HashSet| -> PyResult> { + Ok( + PyFrozenSet::new(py, cache.keys(&keys.iter().cloned().collect::>())?)? + .into_any(), + ) + }; + match name { + "upstream_deps" => Ok(edge_dict(&cache, &self.upstream)?.into_any()), + "downstream_deps" => Ok(edge_dict(&cache, &self.downstream)?.into_any()), + "selected_keys" => key_set(&self.selected), + "user_selected_keys" => key_set(&self.user_selected), + "end_audit_source_keys" => key_set(&self.end_audit_sources), + "execution_order" => Ok(PyTuple::new(py, cache.keys(&self.order)?)?.into_any()), + "all_keys" => { + let values = PyDict::new(py); + for (name, key) in &self.names { + values.set_item(name, cache.key(key)?)?; + } + Ok(values.into_any()) + } + "models_by_name" => { + let values = PyDict::new(py); + for name in &self.model_order { + if let Some(model) = self.models.get(name) { + values.set_item(name, model.bind(py))?; + } + } + Ok(values.into_any()) + } + "python_read_source_names" => Ok(PyFrozenSet::new(py, &self.python_reads)?.into_any()), + "sql_test_selection" => Ok(self.tests.bind(py).clone().into_any()), + "load_sources" => self.load_sources.into_bound_py_any(py), + _ => Err(pyo3::exceptions::PyAttributeError::new_err(name.to_owned())), + } + } +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/carrier.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/carrier.rs new file mode 100644 index 0000000000..92be781877 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/carrier.rs @@ -0,0 +1,19 @@ +//! Carry lone surrogates through the UTF-8 selector grammar as unused private-use characters. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; + +pub(crate) const PRIVATE_USE: std::ops::Range = 0xF0000..0xFFFFE; + +pub(crate) fn ordinal(character: &Bound<'_, PyAny>) -> PyResult { + let code: u32 = character + .py() + .import("builtins")? + .getattr("ord")? + .call1((character,))? + .extract()?; + Ok(code) +} + +pub(crate) fn surrogate_text(py: Python<'_>, code: u32) -> PyResult> { + py.import("builtins")?.getattr("chr")?.call1((code,)) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/mod.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/mod.rs new file mode 100644 index 0000000000..f32364b5d9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod carrier; +pub(crate) mod model_selectors; +pub(crate) mod selectors; +pub(crate) mod unit_tests; diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs new file mode 100644 index 0000000000..b3a9a34c20 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs @@ -0,0 +1,42 @@ +//! `+` marker splitting and SQLBuild-owned model selectors for dbt interop. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; +use pyo3::types::{PyList, PyTuple}; +use sqlbuild_analysis::planner_selection::models::SelectableModel; + +use crate::planner_selection::models::Carrier; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; + +const TAGS_KEY: &str = "tags"; + +const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; + +pub(crate) fn shared_input_error(carrier: &Carrier<'_>, message: &str) -> PyErr { + let raised = imported(carrier.py(), SHARED_EXCEPTIONS_MODULE, "SharedInputError") + .and_then(|class| class.call1((carrier.back(message)?,))); + match raised { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +fn model_tags(model: &Bound<'_, PyAny>) -> PyResult> { + let tags: Bound<'_, PyAny> = config_value(model.py(), &model_values(model)?, TAGS_KEY)?; + if !tags.is_instance_of::() && !tags.is_instance_of::() { + return Ok(Vec::new()); + } + tags.try_iter()?.map(|tag| tag?.str()?.extract()).collect() +} + +pub(crate) fn selectable_model(model: &Bound<'_, PyAny>) -> PyResult { + Ok(SelectableModel { + name: native_field(model, "name")?.extract()?, + tags: model_tags(model)?, + folder: native_field(&native_field(model, "relative_path")?, "parent")? + .call_method0("as_posix")? + .extract()?, + }) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/selectors.rs new file mode 100644 index 0000000000..e464a53b32 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/selectors.rs @@ -0,0 +1,109 @@ +//! Planner selector parsing and resolution over native project graphs, raising `PlannerInputError`. + +use crate::planner_selection::main::resolve_graph_keys::resolve_graph_keys; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::{PyBool, PyFrozenSet}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::errors::SelectorError; +use sqlbuild_analysis::graph::models::{ParsedSelector, ProjectGraph}; + +use crate::graph::main::project::graph_from_view_mappings::graph_from_view_mappings; +use crate::graph::models::GraphViewMappings; +use crate::graph::models::KeyCache; +use crate::graph::models::NativeProjectGraph; +use crate::planner_selection::models::Carrier; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; + +pub(crate) const VIEW_OPTIONS: &[&str] = &[ + "all_keys", + "upstream_deps", + "downstream_deps", + "tag_index", + "path_index", +]; + +/// The parsed selector as Python's `ParsedSelector` or `PathSelector`. +pub(crate) fn parsed_selector<'py>( + carrier: &Carrier<'py>, + parsed: ParsedSelector, +) -> PyResult> { + let py: Python<'py> = carrier.py(); + match parsed { + ParsedSelector::Kind { + kind, + value, + upstream, + downstream, + } => new_record_with( + py, + RecordId::ParsedSelector, + &[ + ("kind", enum_from_value(py, RecordId::SelectorKind, kind)?), + ("value", carrier.back(&value)?.into_any()), + ("upstream", PyBool::new(py, upstream).to_owned().into_any()), + ( + "downstream", + PyBool::new(py, downstream).to_owned().into_any(), + ), + ], + ), + ParsedSelector::Path { + start, + end, + upstream, + downstream, + } => new_record_with( + py, + RecordId::PathSelector, + &[ + ("start_name", carrier.back(&start)?.into_any()), + ("end_name", carrier.back(&end)?.into_any()), + ("upstream", PyBool::new(py, upstream).to_owned().into_any()), + ( + "downstream", + PyBool::new(py, downstream).to_owned().into_any(), + ), + ], + ), + } +} + +pub(crate) fn graph_names(graph: &ProjectGraph) -> Vec { + graph.names.iter().map(|(name, _)| name.clone()).collect() +} + +pub(crate) fn selected<'py>( + carrier: &Carrier<'py>, + outcome: Result, SelectorError>, +) -> PyResult> { + let keys: Vec = outcome.map_err(|error| carrier.error(error))?; + compiled_keys(carrier.py(), &keys) +} + +fn compiled_keys<'py>(py: Python<'py>, keys: &[ObjectKey]) -> PyResult> { + PyFrozenSet::new(py, KeyCache::new(py)?.keys(keys)?) +} + +pub(crate) fn resolved<'py>( + graph: &ProjectGraph, + (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + compiled_keys(select.py(), &resolve_graph_keys(graph, (select, exclude))?) +} + +pub(crate) fn view_graph(mappings: GraphViewMappings<'_, '_>) -> PyResult { + graph_from_view_mappings(mappings) +} + +pub(crate) fn arguments_graph(arguments: &KeywordArguments<'_>) -> PyResult { + view_graph(GraphViewMappings { + all_keys: &arguments.value("all_keys")?, + upstream: &arguments.value("upstream_deps")?, + downstream: &arguments.value("downstream_deps")?, + tag_index: &arguments.value("tag_index")?, + path_index: &arguments.value("path_index")?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs similarity index 83% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs rename to crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs index 7a8c3a69ab..600f939727 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs @@ -1,5 +1,6 @@ //! Unit-test selector splitting over discovered project names. +use crate::planner_selection::main::py_strings::py_strings; use std::collections::BTreeSet; use pyo3::IntoPyObject; @@ -12,8 +13,8 @@ use sqlbuild_analysis::planner_selection::models::{ SelectorGroups, UnitTestSelectorNames, UnitTestSelectorSplit, }; -use crate::bindings::_helpers::compiled_project::job_catalog::job_names; -use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; +use crate::compiled_project::main::job_names::job_names; +use crate::planner_selection::models::Carrier; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::path_stem::path_stem; @@ -115,7 +116,7 @@ fn names_unknown(selectors: &[String], names: &UnitTestSelectorNames) -> bool { }) } -fn split_python<'py>( +pub(crate) fn split_python<'py>( (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), discovered: &Bound<'py, PyAny>, accepts_unit_tests: bool, @@ -175,31 +176,3 @@ fn split_python<'py>( ], ) } - -/// Move unit-test selectors out of resource selectors, or reject them for this command. -pub(crate) fn split_project_unit_test_selectors<'py>( - select: &Bound<'py, PyAny>, - exclude: &Bound<'py, PyAny>, - discovered_inputs: &Bound<'py, PyAny>, - accepts_unit_tests: bool, -) -> PyResult> { - split_python((select, exclude), discovered_inputs, accepts_unit_tests) -} - -/// Split unit-test selectors from resource selectors for a command that runs unit tests. -pub(crate) fn split_unit_test_selectors<'py>( - select: &Bound<'py, PyAny>, - exclude: &Bound<'py, PyAny>, - discovered_inputs: &Bound<'py, PyAny>, -) -> PyResult> { - split_python((select, exclude), discovered_inputs, true) -} - -/// Raise when a selector names a unit test; return the unchanged resource selectors. -pub(crate) fn reject_unit_test_selectors<'py>( - select: &Bound<'py, PyAny>, - exclude: &Bound<'py, PyAny>, - discovered_inputs: &Bound<'py, PyAny>, -) -> PyResult> { - split_python((select, exclude), discovered_inputs, false) -} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/expand_project_upstream_keys.rs b/crates/sqlbuild-python-project/src/planner_selection/main/expand_project_upstream_keys.rs new file mode 100644 index 0000000000..b55375ebfd --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/expand_project_upstream_keys.rs @@ -0,0 +1,14 @@ +//! The upstream key closure of one compiled graph key. + +use crate::graph::main::nodes::closure::closure; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::{PyFrozenSet, PySetMethods}; + +/// The upstream key closure of one compiled graph key. +pub fn expand_project_upstream_keys<'py>( + key: Bound<'py, PyAny>, + upstream_deps: &Bound<'py, PyAny>, +) -> PyResult> { + let visited = closure(vec![key], upstream_deps, (false, None))?; + PyFrozenSet::new(upstream_deps.py(), visited.iter()) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs b/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs new file mode 100644 index 0000000000..fd2804e314 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs @@ -0,0 +1,42 @@ +//! The keys one parsed selector matches before `+` graph expansion. + +use crate::compiled_project::main::graph_with_jobs::graph_with_jobs; +use crate::graph::models::GraphViewMappings; +use crate::graph::models::NativeProjectGraph; +use crate::planner_selection::_helpers::selectors::graph_names; +use crate::planner_selection::_helpers::selectors::selected; +use crate::planner_selection::_helpers::selectors::view_graph; +use crate::planner_selection::models::Carrier; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet, PyString}; +use sqlbuild_analysis::graph::main::match_selector::match_selector; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The keys one parsed selector matches before `+` graph expansion. +pub fn match_project_selector<'py>( + parsed: &Bound<'py, PyAny>, + all_keys: &Bound<'py, PyAny>, + tag_index: &Bound<'py, PyAny>, + path_index: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = parsed.py(); + let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); + let graph: NativeProjectGraph = view_graph(GraphViewMappings { + all_keys, + upstream: &empty, + downstream: &empty, + tag_index, + path_index, + })?; + let value: Bound<'py, PyString> = native_field(parsed, "value")?.downcast_into()?; + let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(&value), || { + Ok(graph_names(graph.graph())) + })?; + let kind: String = native_field(parsed, "kind")?.str()?.extract()?; + let value: String = carrier.send(&value)?; + let graph = graph_with_jobs(py, graph.graph(), &[format!("{kind}:{value}")], &[])?; + selected( + &carrier, + match_selector(&graph, &kind, &value).map(|keys| keys.into_iter().collect()), + ) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/mod.rs b/crates/sqlbuild-python-project/src/planner_selection/main/mod.rs new file mode 100644 index 0000000000..5aada78366 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/mod.rs @@ -0,0 +1,15 @@ +pub mod expand_project_upstream_keys; +pub mod match_project_selector; +pub mod parse_project_selector; +pub mod py_strings; +pub mod register_model_selectors; +pub mod reject_unit_test_selectors; +pub mod resolve_graph_keys; +pub mod resolve_graph_selection; +pub mod resolve_project_selector_tokens; +pub mod resolve_project_selectors; +pub mod resolve_sqlbuild_model_selector_names; +pub mod select_project_graph; +pub mod split_project_unit_test_selectors; +pub mod split_selector_expansion; +pub mod split_unit_test_selectors; diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/parse_project_selector.rs b/crates/sqlbuild-python-project/src/planner_selection/main/parse_project_selector.rs new file mode 100644 index 0000000000..f59594d3c7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/parse_project_selector.rs @@ -0,0 +1,16 @@ +//! Parse one selector token into Python's `ParsedSelector` or `PathSelector`. + +use crate::planner_selection::_helpers::selectors::parsed_selector; +use crate::planner_selection::models::Carrier; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyString; +use sqlbuild_analysis::graph::main::parse_selector::parse_selector; +use sqlbuild_analysis::graph::models::ParsedSelector; + +/// Parse one selector token into Python's `ParsedSelector` or `PathSelector`. +pub fn parse_project_selector<'py>(raw: &Bound<'py, PyString>) -> PyResult> { + let carrier: Carrier<'py> = Carrier::single(raw)?; + let parsed: ParsedSelector = + parse_selector(&carrier.send(raw)?).map_err(|error| carrier.error(error))?; + parsed_selector(&carrier, parsed) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/py_strings.rs b/crates/sqlbuild-python-project/src/planner_selection/main/py_strings.rs new file mode 100644 index 0000000000..9f1fc1e831 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/py_strings.rs @@ -0,0 +1,12 @@ +//! Python strings for each item of an iterable of `str`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyString; + +/// Python strings for each item of an iterable of `str`. +pub fn py_strings<'py>(values: &Bound<'py, PyAny>) -> PyResult>> { + values + .try_iter()? + .map(|value| Ok(value?.downcast_into::()?)) + .collect() +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/register_model_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/main/register_model_selectors.rs new file mode 100644 index 0000000000..0758ef5fcf --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/register_model_selectors.rs @@ -0,0 +1,11 @@ +//! Register the model selectors functions and classes. + +use crate::planner_selection::main::split_selector_expansion::split_selector_expansion; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the model selectors functions and classes. +pub fn register_model_selectors(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(split_selector_expansion, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/reject_unit_test_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/main/reject_unit_test_selectors.rs new file mode 100644 index 0000000000..cc86e70da0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/reject_unit_test_selectors.rs @@ -0,0 +1,13 @@ +//! Raise when a selector names a unit test; return the unchanged resource selectors. + +use crate::planner_selection::_helpers::unit_tests::split_python; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Raise when a selector names a unit test; return the unchanged resource selectors. +pub fn reject_unit_test_selectors<'py>( + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + split_python((select, exclude), discovered_inputs, false) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs new file mode 100644 index 0000000000..c52bfcc3c6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs @@ -0,0 +1,27 @@ +//! Resolve graph keys for native planning without building a public frozenset. + +use crate::compiled_project::main::graph_with_jobs::graph_with_jobs; +use crate::planner_selection::_helpers::selectors::graph_names; +use crate::planner_selection::main::py_strings::py_strings; +use crate::planner_selection::models::Carrier; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; +use sqlbuild_analysis::graph::models::ProjectGraph; + +/// Resolve graph keys for native planning without building a public frozenset. +pub fn resolve_graph_keys<'py>( + graph: &ProjectGraph, + (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = select.py(); + let select: Vec> = py_strings(select)?; + let exclude: Vec> = py_strings(exclude)?; + let texts: Vec> = select.iter().chain(&exclude).cloned().collect(); + let carrier: Carrier<'py> = Carrier::new(py, &texts, || Ok(graph_names(graph)))?; + let (select, exclude) = (carrier.send_all(&select)?, carrier.send_all(&exclude)?); + let all: Vec = select.iter().chain(&exclude).cloned().collect(); + let graph = graph_with_jobs(py, graph, &all, &[])?; + resolve_selectors(&graph, &select, &exclude).map_err(|error| carrier.error(error)) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_selection.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_selection.rs new file mode 100644 index 0000000000..8060ccc44a --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_selection.rs @@ -0,0 +1,15 @@ +//! `--select` minus `--exclude` on a native graph, adding the functions selected models need. + +use crate::graph::models::NativeProjectGraph; +use crate::planner_selection::_helpers::selectors::resolved; +use pyo3::prelude::{Bound, PyAny, PyRef, PyResult}; +use pyo3::types::PyFrozenSet; + +/// `--select` minus `--exclude` on a native graph, adding the functions selected models need. +pub fn resolve_graph_selection<'py>( + graph: PyRef<'py, NativeProjectGraph>, + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, +) -> PyResult> { + resolved(graph.graph(), (select, exclude)) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs new file mode 100644 index 0000000000..b480f0018a --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs @@ -0,0 +1,33 @@ +//! The keys whitespace-separated selector tokens match, without required build resources. + +use crate::compiled_project::main::graph_with_jobs::graph_with_jobs; +use crate::graph::models::NativeProjectGraph; +use crate::planner_selection::_helpers::selectors::VIEW_OPTIONS; +use crate::planner_selection::_helpers::selectors::arguments_graph; +use crate::planner_selection::_helpers::selectors::graph_names; +use crate::planner_selection::_helpers::selectors::selected; +use crate::planner_selection::main::py_strings::py_strings; +use crate::planner_selection::models::Carrier; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet, PyString}; +use sqlbuild_analysis::graph::main::resolve_selector_tokens::resolve_selector_tokens; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// The keys whitespace-separated selector tokens match, without required build resources. +pub fn resolve_project_selector_tokens<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_project_selector_tokens", + options, + (&[&["selectors"], VIEW_OPTIONS].concat(), &[]), + )?; + let graph: NativeProjectGraph = arguments_graph(&arguments)?; + let texts: Vec> = py_strings(&arguments.value("selectors")?)?; + let carrier: Carrier<'py> = Carrier::new(py, &texts, || Ok(graph_names(graph.graph())))?; + let selectors: Vec = carrier.send_all(&texts)?; + let graph = graph_with_jobs(py, graph.graph(), &selectors, &[])?; + selected(&carrier, resolve_selector_tokens(&graph, &selectors)) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selectors.rs new file mode 100644 index 0000000000..6e8d735b6d --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selectors.rs @@ -0,0 +1,27 @@ +//! Model, source, seed and function selectors resolved against compiled graph mappings. + +use crate::graph::models::NativeProjectGraph; +use crate::planner_selection::_helpers::selectors::VIEW_OPTIONS; +use crate::planner_selection::_helpers::selectors::arguments_graph; +use crate::planner_selection::_helpers::selectors::resolved; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet}; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// Model, source, seed and function selectors resolved against compiled graph mappings. +pub fn resolve_project_selectors<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_project_selectors", + options, + (&[&["select", "exclude"], VIEW_OPTIONS].concat(), &[]), + )?; + let graph: NativeProjectGraph = arguments_graph(&arguments)?; + resolved( + graph.graph(), + (&arguments.value("select")?, &arguments.value("exclude")?), + ) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_sqlbuild_model_selector_names.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_sqlbuild_model_selector_names.rs new file mode 100644 index 0000000000..5086df8520 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_sqlbuild_model_selector_names.rs @@ -0,0 +1,42 @@ +//! One SQLBuild-owned model selector core's model names, and its path selector when translated. + +use crate::compiled_views::main::project::compiled_models::compiled_models; +use crate::planner_selection::_helpers::model_selectors::selectable_model; +use crate::planner_selection::models::Carrier; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyString, PyTuple}; +use sqlbuild_analysis::planner_selection::main::sqlbuild_model_selector_names::sqlbuild_model_selector_names; +use sqlbuild_analysis::planner_selection::models::{ModelSelectorNames, SelectableModel}; + +/// One SQLBuild-owned model selector core's model names, and its path selector when translated. +pub fn resolve_sqlbuild_model_selector_names<'py>( + project: &Bound<'py, PyAny>, + term: &Bound<'py, PyString>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let models: Vec = compiled_models(project)? + .try_iter()? + .map(|model| selectable_model(&model?)) + .collect::>()?; + let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(term), || { + Ok(models + .iter() + .flat_map(|model| [model.name.clone(), model.folder.clone()]) + .chain(models.iter().flat_map(|model| model.tags.clone())) + .collect()) + })?; + let resolved: ModelSelectorNames = sqlbuild_model_selector_names(&models, &carrier.send(term)?); + let translated: Option> = resolved + .translated + .as_deref() + .map(|path| carrier.back(path)) + .transpose()?; + PyTuple::new( + py, + [ + PyTuple::new(py, resolved.names)?.into_any(), + translated.into_pyobject(py)?.into_any(), + ], + ) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/select_project_graph.rs b/crates/sqlbuild-python-project/src/planner_selection/main/select_project_graph.rs new file mode 100644 index 0000000000..6bf410c6e5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/select_project_graph.rs @@ -0,0 +1,19 @@ +//! `--select` minus `--exclude` on a project graph, adding the functions selected models need. + +use crate::graph::models::CompiledProjectGraph; +use crate::planner_selection::_helpers::selectors::resolved; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyFrozenSet; + +/// `--select` minus `--exclude` on a project graph, adding the functions selected models need. +pub fn select_project_graph<'py>( + graph: &Bound<'py, PyAny>, + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, +) -> PyResult> { + let graph: &Bound<'py, CompiledProjectGraph> = graph.cast()?; + resolved( + graph.get().native(graph.py()).get().graph(), + (select, exclude), + ) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/split_project_unit_test_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/main/split_project_unit_test_selectors.rs new file mode 100644 index 0000000000..220e84d218 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/split_project_unit_test_selectors.rs @@ -0,0 +1,14 @@ +//! Move unit-test selectors out of resource selectors, or reject them for this command. + +use crate::planner_selection::_helpers::unit_tests::split_python; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Move unit-test selectors out of resource selectors, or reject them for this command. +pub fn split_project_unit_test_selectors<'py>( + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, + accepts_unit_tests: bool, +) -> PyResult> { + split_python((select, exclude), discovered_inputs, accepts_unit_tests) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/split_selector_expansion.rs b/crates/sqlbuild-python-project/src/planner_selection/main/split_selector_expansion.rs new file mode 100644 index 0000000000..1180cfa180 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/split_selector_expansion.rs @@ -0,0 +1,40 @@ +//! Split `+selector+` marker syntax from the selector core without resolving the core. + +use crate::planner_selection::_helpers::model_selectors::shared_input_error; +use crate::planner_selection::models::Carrier; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyBool; +use pyo3::types::PyString; +use pyo3::{IntoPyObject, pyfunction}; +use sqlbuild_analysis::planner_selection::main::split_selector_expansion::split_selector_expansion as split; +use sqlbuild_analysis::planner_selection::models::SelectorExpansion; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::types::RecordId; + +/// Split `+selector+` marker syntax from the selector core without resolving the core. +#[pyfunction] +pub fn split_selector_expansion<'py>(raw: &Bound<'py, PyString>) -> PyResult> { + let py: Python<'py> = raw.py(); + let carrier: Carrier<'py> = Carrier::single(raw)?; + let expansion: SelectorExpansion = + split(&carrier.send(raw)?).map_err(|message| shared_input_error(&carrier, &message))?; + new_record_with( + py, + RecordId::SelectorExpansion, + &[ + ("core", carrier.back(&expansion.core)?.into_any()), + ( + "upstream", + PyBool::new(py, expansion.upstream).to_owned().into_any(), + ), + ( + "downstream", + expansion + .downstream + .into_pyobject(py)? + .to_owned() + .into_any(), + ), + ], + ) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/split_unit_test_selectors.rs b/crates/sqlbuild-python-project/src/planner_selection/main/split_unit_test_selectors.rs new file mode 100644 index 0000000000..c40ac1494a --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/main/split_unit_test_selectors.rs @@ -0,0 +1,13 @@ +//! Split unit-test selectors from resource selectors for a command that runs unit tests. + +use crate::planner_selection::_helpers::unit_tests::split_python; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Split unit-test selectors from resource selectors for a command that runs unit tests. +pub fn split_unit_test_selectors<'py>( + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + split_python((select, exclude), discovered_inputs, true) +} diff --git a/crates/sqlbuild-python-project/src/planner_selection/mod.rs b/crates/sqlbuild-python-project/src/planner_selection/mod.rs new file mode 100644 index 0000000000..4578773f1d --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_selection/mod.rs @@ -0,0 +1,5 @@ +//! Planner selection: selectors, model selectors, unit tests and their carrier. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs b/crates/sqlbuild-python-project/src/planner_selection/models.rs similarity index 68% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs rename to crates/sqlbuild-python-project/src/planner_selection/models.rs index 4d788c3b31..72905eaf73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/models.rs @@ -1,25 +1,39 @@ -//! Carry lone surrogates through the UTF-8 selector grammar as unused private-use characters. - -use std::collections::{BTreeSet, HashSet}; - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; +//! Public records of planner selection. + +use crate::planner_selection::_helpers::carrier::PRIVATE_USE; +use crate::planner_selection::_helpers::carrier::ordinal; +use crate::planner_selection::_helpers::carrier::surrogate_text; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use pyo3::types::PyString; +use pyo3::types::PyStringMethods; use sqlbuild_analysis::graph::errors::SelectorError; - use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; - -const PRIVATE_USE: std::ops::Range = 0xF0000..0xFFFFE; +use std::collections::BTreeSet; +use std::collections::HashSet; /// `str.translate` tables to and from carrier characters; empty when no text holds a surrogate. -pub(crate) struct Carrier<'py> { - py: Python<'py>, - tables: Option<(Bound<'py, PyDict>, Bound<'py, PyDict>)>, +pub struct Carrier<'py> { + pub(crate) py: Python<'py>, + pub(crate) tables: Option<(Bound<'py, PyDict>, Bound<'py, PyDict>)>, +} + +impl std::fmt::Debug for Carrier<'_> { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_struct("Carrier").finish_non_exhaustive() + } } impl<'py> Carrier<'py> { /// A carrier for `texts`, choosing carriers that appear in neither `texts` nor `reserved`. - pub(crate) fn new( + pub fn new( py: Python<'py>, texts: &[Bound<'py, PyString>], reserved: impl FnOnce() -> PyResult>, @@ -65,12 +79,12 @@ impl<'py> Carrier<'py> { } /// A carrier for one text with nothing reserved. - pub(crate) fn single(text: &Bound<'py, PyString>) -> PyResult { + pub fn single(text: &Bound<'py, PyString>) -> PyResult { Self::new(text.py(), std::slice::from_ref(text), || Ok(Vec::new())) } /// The text with surrogates replaced by their carriers. - pub(crate) fn send(&self, text: &Bound<'py, PyString>) -> PyResult { + pub fn send(&self, text: &Bound<'py, PyString>) -> PyResult { match &self.tables { None => Ok(text.to_str()?.to_owned()), Some((send, _)) => text.call_method1("translate", (send,))?.extract(), @@ -78,12 +92,12 @@ impl<'py> Carrier<'py> { } /// The texts with surrogates replaced by their carriers. - pub(crate) fn send_all(&self, texts: &[Bound<'py, PyString>]) -> PyResult> { + pub fn send_all(&self, texts: &[Bound<'py, PyString>]) -> PyResult> { texts.iter().map(|text| self.send(text)).collect() } /// The text with carriers restored to their surrogates. - pub(crate) fn back(&self, text: &str) -> PyResult> { + pub fn back(&self, text: &str) -> PyResult> { let restored: Bound<'py, PyString> = PyString::new(self.py, text); match &self.tables { None => Ok(restored), @@ -93,12 +107,12 @@ impl<'py> Carrier<'py> { } } - pub(crate) fn py(&self) -> Python<'py> { + pub fn py(&self) -> Python<'py> { self.py } /// The `PlannerInputError` instance for a selector failure, with carriers restored. - pub(crate) fn exception(&self, error: SelectorError) -> PyResult> { + pub fn exception(&self, error: SelectorError) -> PyResult> { let keywords: Bound<'py, PyDict> = PyDict::new(self.py); keywords.set_item("code", error.code)?; let help: Option> = error @@ -112,32 +126,10 @@ impl<'py> Carrier<'py> { } /// The `PlannerInputError` to raise for a selector failure, with carriers restored. - pub(crate) fn error(&self, error: SelectorError) -> PyErr { + pub fn error(&self, error: SelectorError) -> PyErr { match self.exception(error) { Ok(instance) => PyErr::from_value(instance), Err(failure) => failure, } } } - -/// Python strings for each item of an iterable of `str`. -pub(crate) fn py_strings<'py>(values: &Bound<'py, PyAny>) -> PyResult>> { - values - .try_iter()? - .map(|value| Ok(value?.downcast_into::()?)) - .collect() -} - -fn ordinal(character: &Bound<'_, PyAny>) -> PyResult { - let code: u32 = character - .py() - .import("builtins")? - .getattr("ord")? - .call1((character,))? - .extract()?; - Ok(code) -} - -fn surrogate_text(py: Python<'_>, code: u32) -> PyResult> { - py.import("builtins")?.getattr("chr")?.call1((code,)) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_completion.rs similarity index 75% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_completion.rs index 48c7fe6b83..6f5ff146a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_completion.rs @@ -1,38 +1,26 @@ //! Relation fixture completion over the Rust fixture planning context. +use crate::planner_strategy::main::complete_fixtures::complete_fixtures; +use crate::planner_strategy::models::FixtureCompletion; use std::collections::HashSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::align_columns::align_columns; use sqlbuild_analysis::planner_fixtures::models::{AlignmentInput, FixtureColumn}; -use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context as project_planning_context; -use crate::bindings::_helpers::planner_strategy::fixture_preparation::{PreparedFixtures, prepare}; -use crate::bindings::_helpers::planner_strategy::fixture_reads::required_fixture_columns; -use crate::bindings::_helpers::planner_strategy::fixture_selection::{ - fixture_keys_requiring_analysis, unknown_fixture_column_diagnostics, -}; -use crate::bindings::_helpers::planner_strategy::fixture_sql::{CompletionSql, render_completed}; -use crate::bindings::_helpers::planner_strategy::fixture_types::expected_fixture_types; -use crate::bindings::_helpers::planner_strategy::fixture_values::{ +use crate::planner_strategy::_helpers::fixtures::fixture_context::planning_context as project_planning_context; +use crate::planner_strategy::_helpers::fixtures::fixture_preparation::PreparedFixtures; +use crate::planner_strategy::_helpers::fixtures::fixture_selection::unknown_fixture_column_diagnostics; +use crate::planner_strategy::_helpers::fixtures::fixture_sql::{CompletionSql, render_completed}; +use crate::planner_strategy::_helpers::fixtures::fixture_values::{ columns, diagnostic, label, record, }; -use crate::bindings::_helpers::planner_strategy::models::{ - ExpectedTypes, RelationFixturePlanningContext, -}; -use crate::bindings::_helpers::planner_strategy::types::FixtureKey; +use crate::planner_strategy::models::{ExpectedTypes, RelationFixturePlanningContext}; +use crate::planner_strategy::types::FixtureKey; use sqlbuild_python_base::records::types::RecordId; -/// Completed fixture SQL and inferred columns by Python fixture key, expected types and diagnostics. -pub(crate) struct FixtureCompletion<'py, 'a> { - pub(crate) sql_by_key: Bound<'py, PyDict>, - pub(crate) inferred_by_fixture: Bound<'py, PyDict>, - pub(crate) expected_types: ExpectedTypes<'a>, - pub(crate) diagnostics: Vec>, -} - impl ExpectedTypes<'_> { /// The relation's expected column types: the chain's propagated types, else the context's. pub(crate) fn get(&self, key: &FixtureKey) -> Option<&[(String, String)]> { @@ -44,21 +32,7 @@ impl ExpectedTypes<'_> { } } -/// Complete the fixture groups of one chain against the planning context. -pub(crate) fn complete_fixtures<'py, 'a>( - context: &'a RelationFixturePlanningContext, - adapter: &Bound<'py, PyAny>, - (ordered_model_names, fixture_groups): (&[String], &Bound<'py, PyAny>), -) -> PyResult> { - let py = adapter.py(); - let prepared = prepare(fixture_groups, context, adapter)?; - let fixture_keys = fixture_keys_requiring_analysis(&prepared, context)?; - let required = required_fixture_columns(context, ordered_model_names, &fixture_keys, adapter)?; - let expected = expected_fixture_types(context, py, ordered_model_names)?; - complete_prepared(prepared, (context, &required), expected, adapter) -} - -fn complete_prepared<'py, 'a>( +pub(crate) fn complete_prepared<'py, 'a>( mut prepared: PreparedFixtures<'py>, (context, required): ( &'a RelationFixturePlanningContext, @@ -199,7 +173,7 @@ fn quoted_names<'py>( /// Complete one chain's fixture groups, building the planning context when none is given. #[pyfunction] #[pyo3(signature = (*, project, adapter, request, planning_context = None))] -fn build_relation_fixture_completion<'py>( +pub(crate) fn build_relation_fixture_completion<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, request: (Vec, Bound<'py, PyAny>), @@ -230,7 +204,3 @@ fn build_relation_fixture_completion<'py>( kwargs.set_item("diagnostics", PyTuple::new(py, completion.diagnostics)?)?; record(py, RecordId::RelationFixtureCompletion, &kwargs) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_relation_fixture_completion, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_context.rs similarity index 79% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_context.rs index 84e858ffdc..e4772f64ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_context.rs @@ -3,41 +3,37 @@ use std::collections::{BTreeSet, HashMap}; use std::sync::{Arc, Mutex}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; -use pyo3::{pyfunction, pymethods, wrap_pyfunction}; +use pyo3::{pyfunction, pymethods}; use sqlbuild_analysis::planner_fixtures::main::fixture_schema::fixture_schema; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; -use crate::bindings::_helpers::analysis_session::fixture_inferences::FixtureColumnInferences; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::native_host::model_lookup::ModelHandle; -use crate::bindings::_helpers::planner_strategy::constants::{ +use crate::analysis_session::models::FixtureColumnInferences; +use crate::compiled_views::main::compiled_models::name_object::name_object; +use crate::compiled_views::main::project::compiled_models::compiled_models; +use crate::compiled_views::main::project::seeds::seeds; +use crate::compiled_views::main::project::sources::sources; +use crate::native_host::models::ModelHandle; +use crate::planner_strategy::_helpers::fixtures::fixture_metadata::model_fixture_relation; +use crate::planner_strategy::_helpers::fixtures::fixture_reads::lineage_reads; +use crate::planner_strategy::_helpers::fixtures::fixture_types::projections; +use crate::planner_strategy::_helpers::fixtures::fixture_values::{columns, folded_name}; +use crate::planner_strategy::constants::{ MODEL_REFERENCE_KIND, SEED_REFERENCE_KIND, SOURCE_REFERENCE_KIND, }; -use crate::bindings::_helpers::planner_strategy::fixture_metadata::model_fixture_relation; -use crate::bindings::_helpers::planner_strategy::fixture_reads::lineage_reads; -use crate::bindings::_helpers::planner_strategy::fixture_types::projections; -use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, folded_name}; -use crate::bindings::_helpers::planner_strategy::models::{ +use crate::planner_strategy::models::{ FixtureReadCacheHost, FixtureRelation, ModelFixtureFacts, RelationFixturePlanningContext, }; -use crate::bindings::_helpers::planner_strategy::types::{FixtureKey, FixtureRelationInput}; +use crate::planner_strategy::types::{FixtureKey, FixtureRelationInput}; +use crate::resource_views::main::resource_reads::name::name; +use crate::resource_views::main::seed_reads::schema_entry::schema_entry; +use crate::resource_views::main::source_reads::source_entry::source_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; static FIXTURE_READS: PyOnceLock> = PyOnceLock::new(); -/// Count one batch resolution of SQL-test model chains. -pub(crate) fn count_chain_resolution(py: Python<'_>) -> PyResult<()> { - work_counts(py)?.borrow_mut().chain_resolutions += 1; - Ok(()) -} - /// The extension-wide fixture read cache and planning work counters. pub(crate) fn work_counts(py: Python<'_>) -> PyResult> { Ok(FIXTURE_READS @@ -60,12 +56,9 @@ pub(crate) fn planning_context( .collect::>()?, None => { let mut models: HashMap = HashMap::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; - models.insert( - model_reads::name_object(&model)?.extract()?, - ModelHandle::read(&model), - ); + models.insert(name_object(&model)?.extract()?, ModelHandle::read(&model)); } models } @@ -79,16 +72,16 @@ pub(crate) fn planning_context( } for (kind, source) in [(SOURCE_REFERENCE_KIND, true), (SEED_REFERENCE_KIND, false)] { let resources: Bound<'_, PyAny> = if source { - project_reads::sources(project)? + sources(project)? } else { - project_reads::seeds(project)? + seeds(project)? }; for resource in resources.try_iter()? { let resource = resource?; let entry: Bound<'_, PyAny> = if source { - source_reads::source_entry(&resource)? + source_entry(&resource)? } else { - seed_reads::schema_entry(&resource)? + schema_entry(&resource)? }; let columns: Vec = columns(&native_field(&entry, "columns")?, false)?; let authoritative = if source { @@ -97,7 +90,7 @@ pub(crate) fn planning_context( !columns.is_empty() }; relations.insert( - (kind.to_owned(), resource_reads::name(&resource)?.extract()?), + (kind.to_owned(), name(&resource)?.extract()?), FixtureRelation { columns, authoritative, @@ -250,7 +243,7 @@ impl RelationFixturePlanningContext { /// Build the project-wide fixture planning context once for many SQL-test plans. #[pyfunction] #[pyo3(signature = (*, project, models_by_name = None))] -fn build_relation_fixture_context( +pub(crate) fn build_relation_fixture_context( project: &Bound<'_, PyAny>, models_by_name: Option>, ) -> PyResult { @@ -260,7 +253,7 @@ fn build_relation_fixture_context( /// Planning contexts built and SQL-test chain batches resolved, optionally resetting both. #[pyfunction] #[pyo3(signature = (*, reset = false))] -fn fixture_planning_work_info(py: Python<'_>, reset: bool) -> PyResult<(u64, u64)> { +pub(crate) fn fixture_planning_work_info(py: Python<'_>, reset: bool) -> PyResult<(u64, u64)> { let host = work_counts(py)?; let mut host = host.borrow_mut(); let counts = (host.context_builds, host.chain_resolutions); @@ -270,9 +263,3 @@ fn fixture_planning_work_info(py: Python<'_>, reset: bool) -> PyResult<(u64, u64 } Ok(counts) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(build_relation_fixture_context, module)?)?; - module.add_function(wrap_pyfunction!(fixture_planning_work_info, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_metadata.rs similarity index 79% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_metadata.rs index 2d15fabc57..fd0d0e684c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_metadata.rs @@ -4,11 +4,12 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_fixtures::main::merge_metadata::merge_metadata; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::model_reads::{inferred_rows, star_flags}; -use crate::bindings::_helpers::planner_strategy::constants::ENFORCED_CONTRACT; -use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, folded_name}; -use crate::bindings::_helpers::planner_strategy::models::FixtureRelation; +use crate::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use crate::compiled_views::main::compiled_models::schema_entry::schema_entry; +use crate::compiled_views::main::compiled_models::star_flags::star_flags; +use crate::planner_strategy::_helpers::fixtures::fixture_values::{columns, folded_name}; +use crate::planner_strategy::constants::ENFORCED_CONTRACT; +use crate::planner_strategy::models::FixtureRelation; use sqlbuild_project::types::InferredRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::main::config::config_value::config_value; @@ -19,7 +20,7 @@ const CONTRACT_KEY: &str = "contract"; /// The model's merged inferred and declared columns, authoritative unless its analysis saw a star. pub(crate) fn model_fixture_relation(model: &Bound<'_, PyAny>) -> PyResult { let inferred: Option> = inferred_rows(model)?; - let schema = model_reads::schema_entry(model)?; + let schema = schema_entry(model)?; let declared: Vec = if schema.is_none() { Vec::new() } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs index 0f5f8af55b..9e313d23bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs @@ -1,13 +1,13 @@ //! Inference and schema preparation at the fixture Python boundary. +use crate::planner_strategy::main::complete_empty_fixture_sql::complete_empty_fixture_sql; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_fixtures::main::empty_fixture_marker::empty_fixture_marker; -use crate::bindings::_helpers::planner_strategy::fixture_sql::complete_empty_fixture_sql; -use crate::bindings::_helpers::planner_strategy::fixture_values::{diagnostic, label}; -use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::planner_strategy::types::FixtureKey; +use crate::planner_strategy::_helpers::fixtures::fixture_values::{diagnostic, label}; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::planner_strategy::types::FixtureKey; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs index 8ac925a58a..8949fddebc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs @@ -1,8 +1,8 @@ //! Boundary conversion for native fixture read analysis. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyFrozenSet; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::required_columns::required_columns; use sqlbuild_analysis::planner_fixtures::main::star_keys::star_keys; use sqlbuild_analysis::planner_fixtures::models::{ @@ -13,14 +13,16 @@ use sqlbuild_analysis::required_columns::models::ColumnRead; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use std::collections::{HashMap, HashSet}; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::model_reads::{ - LineageFactRow, lineage_facts, proven_inputs, star_flags, -}; -use crate::bindings::_helpers::planner_strategy::constants::HIGH_CONFIDENCE; -use crate::bindings::_helpers::planner_strategy::fixture_context::work_counts; -use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::planner_strategy::types::FixtureKey; +use crate::compiled_views::main::compiled_models::dep_rows::dep_rows; +use crate::compiled_views::main::compiled_models::lineage_facts::lineage_facts; +use crate::compiled_views::main::compiled_models::proven_inputs::proven_inputs; +use crate::compiled_views::main::compiled_models::query_sql::query_sql; +use crate::compiled_views::main::compiled_models::star_flags::star_flags; +use crate::compiled_views::models::LineageFactRow; +use crate::planner_strategy::_helpers::fixtures::fixture_context::work_counts; +use crate::planner_strategy::constants::HIGH_CONFIDENCE; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::planner_strategy::types::FixtureKey; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; @@ -62,7 +64,10 @@ pub(crate) fn required_fixture_columns( continue; }; let mut references: Vec = Vec::new(); - for reference in model_reads::references(&model)?.try_iter()? { + for reference in + crate::compiled_views::main::compiled_models::references::references(&model)? + .try_iter()? + { let reference = reference?; references.push(FixtureReadReference { kind: native_field(&reference, "ref_kind")?.extract()?, @@ -74,7 +79,7 @@ pub(crate) fn required_fixture_columns( models.push(FixtureReadModel { name: name.clone(), key: FixtureReadKey { - sql: model_reads::query_sql(&model)?.extract()?, + sql: query_sql(&model)?.extract()?, references, dialect: dialect.clone(), }, @@ -132,7 +137,7 @@ fn model_stars( return Ok(Vec::new()); } let mut deps: Vec<(String, String)> = Vec::new(); - for key in model_reads::dep_rows(model)? { + for key in dep_rows(model)? { if schema.iter().any(|relation| relation.key == key) { deps.push(key); } @@ -159,7 +164,7 @@ fn model_stars( } else { None }; - let sql: String = model_reads::query_sql(model)?.extract()?; + let sql: String = query_sql(model)?.extract()?; model .py() .compiler_detach(|| { @@ -173,14 +178,10 @@ fn model_stars( .map_err(value_error) } -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(fixture_read_cache_info, module)?) -} - /// Report resolver misses and retained entries, optionally clearing prior fixture reuse. #[pyfunction] #[pyo3(signature = (*, reset = false))] -fn fixture_read_cache_info(py: pyo3::Python<'_>, reset: bool) -> PyResult<(u64, usize)> { +pub(crate) fn fixture_read_cache_info(py: pyo3::Python<'_>, reset: bool) -> PyResult<(u64, usize)> { let owner = work_counts(py)?; let mut owner = owner.borrow_mut(); if reset { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_selection.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_selection.rs index 3f25962c17..7ab6d934ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_selection.rs @@ -9,10 +9,10 @@ use sqlbuild_analysis::planner_fixtures::main::{ unknown_columns::unknown_columns, }; -use crate::bindings::_helpers::planner_strategy::fixture_preparation::PreparedFixtures; -use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, diagnostic, label}; -use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::planner_strategy::types::FixtureKey; +use crate::planner_strategy::_helpers::fixtures::fixture_preparation::PreparedFixtures; +use crate::planner_strategy::_helpers::fixtures::fixture_values::{columns, diagnostic, label}; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::planner_strategy::types::FixtureKey; /// Fixtures whose inferred columns do not already match an authoritative relation shape. pub(crate) fn fixture_keys_requiring_analysis( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs similarity index 52% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs index a3b237a8d3..f2ff9d0dba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs @@ -3,14 +3,13 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::planner_fixtures::main::{ - empty_fixture_error::empty_fixture_error, empty_fixture_marker::empty_fixture_marker, - render_completed::render_completed as native_render_completed, render_empty::render_empty, + empty_fixture_marker::empty_fixture_marker, + render_completed::render_completed as native_render_completed, }; use sqlbuild_analysis::planner_fixtures::models::{ FixtureColumn, FixtureSqlInput, SuppliedProjection, }; -use crate::bindings::_helpers::planner_strategy::models::FixtureRelation; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -18,33 +17,6 @@ pub(crate) fn fixture_marker(sql: &str) -> PyResult { empty_fixture_marker(sql).map_err(value_error) } -/// An empty-fixture marker rendered as the relation's typed empty projection, or why it cannot be. -pub(crate) fn complete_empty_fixture_sql( - sql: &str, - relation: Option<&FixtureRelation>, - adapter: &Bound<'_, PyAny>, -) -> PyResult<(String, Option)> { - if !fixture_marker(sql)? { - return Ok((sql.into(), None)); - } - let authoritative = relation.is_some_and(|relation| relation.authoritative); - let metadata: &[FixtureColumn] = match relation { - Some(relation) if authoritative => &relation.columns, - _ => &[], - }; - if let Some(error) = empty_fixture_error(authoritative, metadata) { - return Ok((sql.into(), Some(error))); - } - Ok(( - render_empty(metadata, |name| { - adapter - .call_method1("render_identifier", (name,))? - .extract::() - })?, - None, - )) -} - pub(crate) struct CompletionSql<'py, 'a> { pub(crate) sql: &'a str, pub(crate) inferred: Bound<'py, PyAny>, @@ -95,34 +67,3 @@ pub(crate) fn render_completed( }, ) } - -/// Fixture SQL with its untyped null columns cast to the expected types. -pub(crate) fn complete_typed_null_columns( - sql: &str, - inference: &Bound<'_, PyAny>, - expected_types: &[(String, String)], - adapter: &Bound<'_, PyAny>, -) -> PyResult { - let py = inference.py(); - let typed_nulls = PyDict::new(py); - for name in native_field(inference, "null_literal_names")?.try_iter()? { - let name = name?; - let folded: String = name.extract()?; - if let Some((_, expected)) = expected_types.iter().find(|(column, _)| *column == folded) { - typed_nulls.set_item(name, expected)?; - } - } - if typed_nulls.is_empty() { - return Ok(sql.into()); - } - render_completed( - CompletionSql { - sql, - inferred: native_field(inference, "columns")?, - typed_nulls, - quoted_names: native_field(inference, "quoted_names")?, - completed: Vec::new(), - }, - adapter, - ) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_types.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_types.rs index 17109ed3ce..4222c87804 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_types.rs @@ -9,14 +9,11 @@ use sqlbuild_analysis::planner_fixtures::models::{ use sqlbuild_core::text::main::python_casefold::python_casefold; -use crate::bindings::_helpers::compiled_views::model_reads::{LineageFactRow, lineage_facts}; -use crate::bindings::_helpers::planner_strategy::constants::{ - DIRECT_TRANSFORM, MODEL_REFERENCE_KIND, -}; -use crate::bindings::_helpers::planner_strategy::models::{ - ExpectedTypes, RelationFixturePlanningContext, -}; -use crate::bindings::_helpers::planner_strategy::types::FixtureKey; +use crate::compiled_views::main::compiled_models::lineage_facts::lineage_facts; +use crate::compiled_views::models::LineageFactRow; +use crate::planner_strategy::constants::{DIRECT_TRANSFORM, MODEL_REFERENCE_KIND}; +use crate::planner_strategy::models::{ExpectedTypes, RelationFixturePlanningContext}; +use crate::planner_strategy::types::FixtureKey; /// Each chain model's known column types with direct lineage types propagated from its inputs. pub(crate) fn expected_fixture_types<'a>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_values.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs rename to crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_values.rs diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/mod.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/mod.rs new file mode 100644 index 0000000000..1f9df7eb9f --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod fixture_completion; +pub(crate) mod fixture_context; +pub(crate) mod fixture_metadata; +pub(crate) mod fixture_preparation; +pub(crate) mod fixture_reads; +pub(crate) mod fixture_selection; +pub(crate) mod fixture_sql; +pub(crate) mod fixture_types; +pub(crate) mod fixture_values; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/_helpers/mod.rs b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/mod.rs new file mode 100644 index 0000000000..1d7e992f42 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod fixtures; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/constants.rs b/crates/sqlbuild-python-project/src/planner_strategy/constants.rs new file mode 100644 index 0000000000..420e5189c5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/constants.rs @@ -0,0 +1,52 @@ +//! Public constants of planner strategy. + +pub const AUDIT_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; + +pub const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; + +pub const MODEL_ATTACHMENT: &str = "model"; + +pub const SOURCE_ATTACHMENT: &str = "source"; + +pub const INCREMENTAL_MATERIALIZATION: &str = "incremental"; + +pub const MICROBATCH_MODE: &str = "microbatch"; + +pub const CUSTOM_MATERIALIZATION: &str = "custom"; + +pub const ENFORCED_CONTRACT: &str = "enforced"; + +pub const TIMESTAMP_CURSOR: &str = "timestamp"; + +pub const MODEL_REFERENCE: &str = "ref"; + +pub const SEED_REFERENCE: &str = "seed"; + +pub const SOURCE_REFERENCE: &str = "source"; + +pub const DDL_SEPARATOR: &str = ";\n\n"; + +/// The lineage confidence whose reads fixtures must provide. +pub(crate) const HIGH_CONFIDENCE: &str = "high"; + +/// The lineage transform kind that passes an input column's type through. +pub(crate) const DIRECT_TRANSFORM: &str = "direct"; + +/// Resource type values keying fixture relations. +pub const MODEL_REFERENCE_KIND: &str = "model"; + +pub(crate) const SOURCE_REFERENCE_KIND: &str = "source"; + +pub(crate) const SEED_REFERENCE_KIND: &str = "seed"; + +/// The SQL analysis dialect whose adapter lays tables out by the physical storage keys. +pub const PHYSICAL_STORAGE_DIALECT: &str = "clickhouse"; + +/// The error code for physical storage keys on an adapter without them. +pub const PHYSICAL_STORAGE_UNSUPPORTED_CODE: &str = "A011"; + +/// The error code for `replace_key` on an adapter that cannot replace groups of rows. +pub const GROUP_REPLACEMENT_UNSUPPORTED_CODE: &str = "A012"; + +/// Plan action values that create the model's table anew or leave it untouched. +pub const TABLE_RECREATING_ACTIONS: [&str; 2] = ["create_table", "skip"]; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs new file mode 100644 index 0000000000..fe7dffadd8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs @@ -0,0 +1,36 @@ +//! An empty-fixture marker rendered as the relation's typed empty projection, or why it cannot be. + +use crate::planner_strategy::_helpers::fixtures::fixture_sql::fixture_marker; +use crate::planner_strategy::models::FixtureRelation; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::planner_fixtures::main::{ + empty_fixture_error::empty_fixture_error, render_empty::render_empty, +}; +use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; + +/// An empty-fixture marker rendered as the relation's typed empty projection, or why it cannot be. +pub fn complete_empty_fixture_sql( + sql: &str, + relation: Option<&FixtureRelation>, + adapter: &Bound<'_, PyAny>, +) -> PyResult<(String, Option)> { + if !fixture_marker(sql)? { + return Ok((sql.into(), None)); + } + let authoritative = relation.is_some_and(|relation| relation.authoritative); + let metadata: &[FixtureColumn] = match relation { + Some(relation) if authoritative => &relation.columns, + _ => &[], + }; + if let Some(error) = empty_fixture_error(authoritative, metadata) { + return Ok((sql.into(), Some(error))); + } + Ok(( + render_empty(metadata, |name| { + adapter + .call_method1("render_identifier", (name,))? + .extract::() + })?, + None, + )) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs new file mode 100644 index 0000000000..2edf3b0b59 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_fixtures.rs @@ -0,0 +1,24 @@ +//! Complete the fixture groups of one chain against the planning context. + +use crate::planner_strategy::_helpers::fixtures::fixture_completion::complete_prepared; +use crate::planner_strategy::_helpers::fixtures::fixture_preparation::prepare; +use crate::planner_strategy::_helpers::fixtures::fixture_reads::required_fixture_columns; +use crate::planner_strategy::_helpers::fixtures::fixture_selection::fixture_keys_requiring_analysis; +use crate::planner_strategy::_helpers::fixtures::fixture_types::expected_fixture_types; +use crate::planner_strategy::models::FixtureCompletion; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Complete the fixture groups of one chain against the planning context. +pub fn complete_fixtures<'py, 'a>( + context: &'a RelationFixturePlanningContext, + adapter: &Bound<'py, PyAny>, + (ordered_model_names, fixture_groups): (&[String], &Bound<'py, PyAny>), +) -> PyResult> { + let py = adapter.py(); + let prepared = prepare(fixture_groups, context, adapter)?; + let fixture_keys = fixture_keys_requiring_analysis(&prepared, context)?; + let required = required_fixture_columns(context, ordered_model_names, &fixture_keys, adapter)?; + let expected = expected_fixture_types(context, py, ordered_model_names)?; + complete_prepared(prepared, (context, &required), expected, adapter) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs new file mode 100644 index 0000000000..99860782a3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/complete_typed_null_columns.rs @@ -0,0 +1,38 @@ +//! Fixture SQL with its untyped null columns cast to the expected types. + +use crate::planner_strategy::_helpers::fixtures::fixture_sql::CompletionSql; +use crate::planner_strategy::_helpers::fixtures::fixture_sql::render_completed; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Fixture SQL with its untyped null columns cast to the expected types. +pub fn complete_typed_null_columns( + sql: &str, + inference: &Bound<'_, PyAny>, + expected_types: &[(String, String)], + adapter: &Bound<'_, PyAny>, +) -> PyResult { + let py = inference.py(); + let typed_nulls = PyDict::new(py); + for name in native_field(inference, "null_literal_names")?.try_iter()? { + let name = name?; + let folded: String = name.extract()?; + if let Some((_, expected)) = expected_types.iter().find(|(column, _)| *column == folded) { + typed_nulls.set_item(name, expected)?; + } + } + if typed_nulls.is_empty() { + return Ok(sql.into()); + } + render_completed( + CompletionSql { + sql, + inferred: native_field(inference, "columns")?, + typed_nulls, + quoted_names: native_field(inference, "quoted_names")?, + completed: Vec::new(), + }, + adapter, + ) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/count_chain_resolution.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/count_chain_resolution.rs new file mode 100644 index 0000000000..d743a78161 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/count_chain_resolution.rs @@ -0,0 +1,10 @@ +//! Count one batch resolution of SQL-test model chains. + +use crate::planner_strategy::_helpers::fixtures::fixture_context::work_counts; +use pyo3::prelude::{PyResult, Python}; + +/// Count one batch resolution of SQL-test model chains. +pub fn count_chain_resolution(py: Python<'_>) -> PyResult<()> { + work_counts(py)?.borrow_mut().chain_resolutions += 1; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/mod.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/mod.rs new file mode 100644 index 0000000000..5cacd8332e --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/mod.rs @@ -0,0 +1,8 @@ +pub mod complete_empty_fixture_sql; +pub mod complete_fixtures; +pub mod complete_typed_null_columns; +pub mod count_chain_resolution; +pub mod planning_context; +pub mod register_fixture_completion; +pub mod register_fixture_context; +pub mod register_fixture_reads; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/planning_context.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/planning_context.rs new file mode 100644 index 0000000000..c9861c24fb --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/planning_context.rs @@ -0,0 +1,15 @@ +//! The planning context of a project, over its models or the given name-to-model map. + +use crate::planner_strategy::_helpers::fixtures::fixture_context; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyDict; + +/// The planning context of a project, over its models or the given name-to-model map. +#[inline] +pub fn planning_context( + project: &Bound<'_, PyAny>, + models_by_name: Option>, +) -> PyResult { + fixture_context::planning_context(project, models_by_name) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_completion.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_completion.rs new file mode 100644 index 0000000000..d1a13f6ec3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_completion.rs @@ -0,0 +1,10 @@ +//! Register the fixture completion functions and classes. + +use crate::planner_strategy::_helpers::fixtures::fixture_completion::build_relation_fixture_completion; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the fixture completion functions and classes. +pub fn register_fixture_completion(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_relation_fixture_completion, module)?) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs new file mode 100644 index 0000000000..6107ba4a68 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs @@ -0,0 +1,14 @@ +//! Register the fixture context functions and classes. + +use crate::planner_strategy::_helpers::fixtures::fixture_context::build_relation_fixture_context; +use crate::planner_strategy::_helpers::fixtures::fixture_context::fixture_planning_work_info; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the fixture context functions and classes. +pub fn register_fixture_context(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(build_relation_fixture_context, module)?)?; + module.add_function(wrap_pyfunction!(fixture_planning_work_info, module)?) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_reads.rs b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_reads.rs new file mode 100644 index 0000000000..d57bb7ebd9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_reads.rs @@ -0,0 +1,10 @@ +//! Register the fixture reads functions and classes. + +use crate::planner_strategy::_helpers::fixtures::fixture_reads::fixture_read_cache_info; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the fixture reads functions and classes. +pub fn register_fixture_reads(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(fixture_read_cache_info, module)?) +} diff --git a/crates/sqlbuild-python-project/src/planner_strategy/mod.rs b/crates/sqlbuild-python-project/src/planner_strategy/mod.rs new file mode 100644 index 0000000000..d4dfb24511 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/mod.rs @@ -0,0 +1,7 @@ +//! Planner strategy for fixtures: context, metadata, preparation, reads, selection, SQL and values. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/planner_strategy/models.rs b/crates/sqlbuild-python-project/src/planner_strategy/models.rs new file mode 100644 index 0000000000..7895145136 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_strategy/models.rs @@ -0,0 +1,126 @@ +//! Public records of planner strategy. + +use crate::graph::models::NativeProjectGraph; +use crate::native_host::models::ModelHandle; +use crate::native_host::models::ModelLookup; +use crate::planner_strategy::types::FixtureKey; +use pyo3::prelude::{Bound, Py}; +use pyo3::pyclass; +use pyo3::types::PyDict; +use sqlbuild_analysis::planner_fixtures::models::{ + FixtureColumn, FixtureReadCache, FixtureTypeProjection, +}; +use sqlbuild_analysis::required_columns::models::ColumnRead; +use std::collections::{BTreeSet, HashMap}; +use std::sync::{Arc, Mutex}; + +/// Completed fixture SQL and inferred columns by Python fixture key, expected types and diagnostics. +#[derive(Debug)] +pub struct FixtureCompletion<'py, 'a> { + pub sql_by_key: Bound<'py, PyDict>, + pub inferred_by_fixture: Bound<'py, PyDict>, + pub(crate) expected_types: ExpectedTypes<'a>, + pub diagnostics: Vec>, +} + +/// Per-extension cache replacing the Python fixture resolver's LRU. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct FixtureReadCacheHost { + pub cache: FixtureReadCache, + /// Fixture planning contexts built since the last reset. + pub context_builds: u64, + /// SQL-test model chain batches resolved since the last reset. + pub chain_resolutions: u64, +} + +/// One relation's known fixture columns and whether they form an authoritative shape. +#[derive(Clone, Debug)] +pub struct FixtureRelation { + pub columns: Vec, + pub authoritative: bool, +} + +/// One model's high-confidence lineage reads and direct type projections. +#[derive(Debug)] +pub(crate) struct ModelFixtureFacts { + pub reads: Vec, + pub projections: Vec, +} + +/// Project-wide fixture metadata shared across test and scenario planning. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct RelationFixturePlanningContext { + pub models_by_name: HashMap, + pub relations: HashMap, + pub authoritative_columns: HashMap>, + pub expected_types: HashMap>, + pub fixture_inferences: Py, + pub(crate) model_facts: Mutex>>, +} + +/// The expected column types of one completion: its chain's propagated types over the context's. +#[derive(Debug)] +pub(crate) struct ExpectedTypes<'a> { + pub overlay: Vec<(FixtureKey, Vec<(String, String)>)>, + pub base: &'a HashMap>, +} + +/// One model's resolved plan decisions, assembled into its `ModelPlanEntry`. +#[derive(Debug)] +pub struct ModelEntryDecisions<'py> { + pub model: Bound<'py, pyo3::types::PyAny>, + pub adapter: Bound<'py, pyo3::types::PyAny>, + pub snapshot: Bound<'py, pyo3::types::PyAny>, + pub change_result: Bound<'py, pyo3::types::PyAny>, + pub action: Bound<'py, pyo3::types::PyAny>, + pub reason: Bound<'py, pyo3::types::PyAny>, + pub materialization_type: Bound<'py, pyo3::types::PyAny>, + pub resolved_sql: Bound<'py, pyo3::types::PyAny>, + pub sql_analysis_enabled: bool, + pub hooks: ( + Bound<'py, pyo3::types::PyAny>, + Bound<'py, pyo3::types::PyAny>, + ), + pub cursor_column: Bound<'py, pyo3::types::PyAny>, + pub cursor_input_relations: Bound<'py, pyo3::types::PyAny>, + pub runtime_owned_cursor_bounds: bool, + pub cursor_bounds: Bound<'py, pyo3::types::PyAny>, + pub microbatch_range: Bound<'py, pyo3::types::PyAny>, + pub empty_cursor_inputs: Bound<'py, pyo3::types::PyAny>, + pub waiting_on_empty_inputs: bool, + pub backfill: Bound<'py, pyo3::types::PyAny>, + pub cursor_overrides: ( + Bound<'py, pyo3::types::PyAny>, + Bound<'py, pyo3::types::PyAny>, + ), + pub cursor_configs: ( + Bound<'py, pyo3::types::PyAny>, + Bound<'py, pyo3::types::PyAny>, + ), + pub invocation_time: Bound<'py, pyo3::types::PyAny>, +} + +/// The project lookups that resolve one model's cursor inputs. +#[derive(Debug)] +pub struct CursorInputLookups<'py> { + pub models_by_name: ModelLookup, + pub functions_by_name: Bound<'py, pyo3::types::PyAny>, + pub source_map: Bound<'py, pyo3::types::PyAny>, + pub source_warehouse_columns: Bound<'py, pyo3::types::PyAny>, + pub model_locations: Bound<'py, pyo3::types::PyAny>, + pub seed_locations: Bound<'py, pyo3::types::PyAny>, + pub runtime_cursor_producer_names: Bound<'py, pyo3::types::PyAny>, +} + +/// The relations and lineage that render and schedule selected audits. +#[derive(Debug)] +pub struct AuditPlanning<'py> { + pub model_locations: Bound<'py, pyo3::types::PyAny>, + pub seed_locations: Bound<'py, pyo3::types::PyAny>, + pub source_map: Bound<'py, pyo3::types::PyAny>, + pub adapter: Bound<'py, pyo3::types::PyAny>, + pub model_materializations: Bound<'py, pyo3::types::PyAny>, + pub lineage: Py, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/types.rs b/crates/sqlbuild-python-project/src/planner_strategy/types.rs similarity index 72% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/types.rs rename to crates/sqlbuild-python-project/src/planner_strategy/types.rs index 2ae899fbc1..b631369d0b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/types.rs +++ b/crates/sqlbuild-python-project/src/planner_strategy/types.rs @@ -1,7 +1,7 @@ -//! Fixture planning key and input shapes. +//! Public traits and type aliases of planner strategy. /// `(resource type value, name)` of one fixture relation. -pub(crate) type FixtureKey = (String, String); +pub type FixtureKey = (String, String); /// One relation's `((name, type, nullable), ...)` columns and whether they are authoritative. pub(crate) type FixtureRelationInput = (Vec<(String, Option, Option)>, bool); diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/mod.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/mod.rs new file mode 100644 index 0000000000..a397cc9af2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod snapshot; diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs new file mode 100644 index 0000000000..2f10cb6b41 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod snapshot_boundary; +pub(crate) mod snapshot_declarations; +pub(crate) mod snapshot_execution; +pub(crate) mod snapshot_models; +pub(crate) mod snapshot_project; +pub(crate) mod snapshot_resources; +pub(crate) mod snapshot_roles; +pub(crate) mod snapshot_worker; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs similarity index 91% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs rename to crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs index ffb48a60a0..38373ed435 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_boundary.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_boundary.rs @@ -1,5 +1,6 @@ //! Discovery files and authored test payloads remain the Python host's boundary. +use crate::project_snapshot::main::relative_path::relative_path; use std::collections::BTreeMap; use std::fs::File; use std::io::Read; @@ -12,13 +13,13 @@ use serde_json::{Map, Value, json}; use sha2::{Digest, Sha256}; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::snapshot::{destination, insert_node}; -use crate::bindings::_helpers::compiled_project::values::plain_value; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::compiled_views::seed_views::{NativeCompiledSeed, SeedViews}; +use crate::compiled_project::main::plain_value::plain_value; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::NativeProjectView; +use crate::project_snapshot::_helpers::snapshot::snapshot_project::{destination, insert_node}; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::types::SeedViews; use sqlbuild_python_base::records::main::fields::field::field; -use sqlbuild_python_base::records::main::fields::path_posix::path_posix; /// Read only declared metadata fields, never callable attributes or connection settings. pub(crate) fn metadata(value: &Bound<'_, PyAny>, names: &[&str]) -> PyResult { @@ -29,10 +30,6 @@ pub(crate) fn metadata(value: &Bound<'_, PyAny>, names: &[&str]) -> PyResult) -> PyResult { - path_posix(&field(value, "relative_path")?) -} - /// Source locations are local compiler diagnostics, not relocatable column metadata. pub(crate) fn columns(value: &Bound<'_, PyAny>) -> PyResult> { value diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs rename to crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs index f0d6e873c0..a628edd529 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs @@ -11,7 +11,6 @@ use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_discovery::orchestration_files::models::{ FilePosition, InitialStatus, NamedModule, OrchestrationInventory, RunFile, ScheduleSource, }; -use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_orchestration::declarations::main::group_dependencies::group_dependencies; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ @@ -20,12 +19,11 @@ use sqlbuild_orchestration::declarations::models::{ }; use sqlbuild_orchestration::snapshots::models::SnapshotInput; -use crate::bindings::_helpers::graph::python_selection::ProjectSelection; -use crate::bindings::_helpers::native_host::orchestration_triggers::{ - CalendarRequest, compile_calendars, compile_sensors, -}; +use crate::graph::models::ProjectSelection; +use crate::native_host::main::compile_calendars::compile_calendars; +use crate::native_host::main::compile_sensors::compile_sensors; +use crate::native_host::models::CalendarRequest; use sqlbuild_python_base::orchestration_reads::main::imported_declarations::imported_declarations; -use sqlbuild_python_base::orchestration_reads::main::module_declarations::module_declarations; use sqlbuild_python_base::orchestration_reads::main::run_controls::run_controls; use sqlbuild_python_base::orchestration_reads::main::run_resources::run_resources; use sqlbuild_python_base::orchestration_reads::models::ImportedDeclarations; @@ -33,25 +31,6 @@ use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_ use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; use sqlbuild_rules::lint_text::models::FormatSettings; -/// The calendars and sensors of the inventory's modules, imported by publication and by sensor evaluation. -pub(crate) fn python_declarations<'py>( - py: Python<'py>, - project_dir: &Path, - inventory: &OrchestrationInventory, -) -> PyResult>> { - let tree = ProjectTree::new(project_dir); - let mut declarations: Vec> = Vec::new(); - let mut seen: BTreeSet = BTreeSet::new(); - for module in inventory.calendars.iter().chain(&inventory.sensors) { - for declaration in module_declarations(py, &tree.absolute(&module.path), project_dir)? { - if seen.insert(declaration.as_ptr() as usize) { - declarations.push(declaration); - } - } - } - Ok(declarations) -} - /// Where a TOML job or group is declared: its file and the line of its `name`. fn file_site(path: &str, position: FilePosition) -> Option { Some(DeclaredAt { diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_execution.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_execution.rs new file mode 100644 index 0000000000..1e3936ef2f --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_execution.rs @@ -0,0 +1,32 @@ +//! Frozen selection, executor callbacks and completion belong to one verified worker build. + +use pyo3::exceptions::PyRuntimeError; +use sqlbuild_core::models::CompiledResourceType; +use sqlbuild_orchestration::runs::models::NodeOutcome; + +/// A finished node's outcome, rows affected, error code and error message. +pub(crate) type Completion = (NodeOutcome, Option, (Option, Option)); + +/// Failure outranks cancellation, then warnings, then success. +pub(crate) fn severity(outcome: NodeOutcome) -> u8 { + match outcome { + NodeOutcome::Skipped => 0, + NodeOutcome::Success => 1, + NodeOutcome::Warn => 2, + NodeOutcome::Cancelled => 3, + NodeOutcome::Failed => 4, + } +} + +/// Whether a snapshot node key names a scalar or table function. +pub(crate) fn is_function(key: &str) -> bool { + key.split_once(':').is_some_and(|(kind, _)| { + [CompiledResourceType::Udf, CompiledResourceType::TableFn] + .iter() + .any(|function| function.as_str() == kind) + }) +} + +pub(crate) fn poisoned() -> pyo3::PyErr { + PyRuntimeError::new_err("worker build outcome lock is unavailable") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs rename to crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs index 76bdb01204..44aeaa81e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_models.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_models.rs @@ -13,13 +13,13 @@ use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; use sqlbuild_analysis::sql_macros::main::typed_analysis_sql::typed_analysis_sql; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::snapshot::{ +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::models::NativeProjectView; +use crate::compiled_views::types::ModelInputRecord; +use crate::project_snapshot::_helpers::snapshot::snapshot_project::{ dependency_keys, destination, insert_node, }; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::model_input_view::ModelInputRecord; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use sqlbuild_python_front::attachments::main::analysis_cursor_splices::analysis_cursor_splices; /// Each model's snapshot node, and the `columns` document of ranges kept out of node identity. diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_project.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_project.rs new file mode 100644 index 0000000000..d1491c4fe9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_project.rs @@ -0,0 +1,40 @@ +//! Project metadata stays native until its immutable snapshot is sent over HTTP. + +use std::collections::BTreeMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::PyResult; +use serde_json::Value; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; + +/// Duplicate resource identities must fail, rather than silently replacing an earlier node. +pub(crate) fn insert_node( + mut nodes: BTreeMap, + key: String, + node: SnapshotNode, +) -> PyResult> { + if nodes.contains_key(&key) { + return Err(PyValueError::new_err(format!( + "duplicate snapshot node identity {key}" + ))); + } + nodes.insert(key, node); + Ok(nodes) +} + +pub(crate) fn dependency_keys(deps: &[(String, String)]) -> Vec { + deps.iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect() +} + +pub(crate) fn destination(value: &sqlbuild_project::models::DestinationFact) -> Value { + serde_json::json!({ + "database": value.database, + "schema": value.schema, + "name": value.name, + "qualified_name": value.qualified_name, + "logical_database": value.logical_database, + "logical_schema": value.logical_schema, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs rename to crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs index 959edffc0e..c7791c304a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_resources.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_resources.rs @@ -1,5 +1,6 @@ //! Sources, functions and audits are read from their retained native records. +use crate::project_snapshot::main::relative_path::relative_path; use std::collections::BTreeMap; use pyo3::prelude::{PyAnyMethods, PyResult, Python}; @@ -7,16 +8,16 @@ use serde_json::{Value, json}; use sqlbuild_core::nested_progress::main::audit_resource_id::audit_resource_id; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::snapshot::{ - dependency_keys, destination, insert_node, +use crate::compiled_project::main::plain_value::plain_value; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::NativeProjectView; +use crate::project_snapshot::_helpers::snapshot::snapshot_boundary::{ + boundary_nodes, columns, metadata, }; -use crate::bindings::_helpers::compiled_project::snapshot_boundary::{ - boundary_nodes, columns, metadata, relative_path, +use crate::project_snapshot::_helpers::snapshot::snapshot_project::{ + dependency_keys, destination, insert_node, }; -use crate::bindings::_helpers::compiled_project::values::plain_value; -use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::resource_views::models::NativeCompiledFunction; pub(crate) fn resource_nodes( py: Python<'_>, diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_roles.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_roles.rs new file mode 100644 index 0000000000..79abee250c --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_roles.rs @@ -0,0 +1,4 @@ +//! Project roles from the reviewed `roles.toml` file, their selectors resolved against this snapshot. + +/// The project-root file that defines project roles. +pub(crate) const ROLES_FILE: &str = "roles.toml"; diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_worker.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_worker.rs new file mode 100644 index 0000000000..e8733cbed4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_worker.rs @@ -0,0 +1,24 @@ +//! A worker interpreter retains the native claim owner; it never opens the control database. + +use pyo3::exceptions::PyRuntimeError; +use sqlbuild_orchestration::worker::models::{WorkerClient, WorkerSession}; + +#[derive(Debug)] +pub(crate) struct WorkerOwner { + pub(crate) runtime: tokio::runtime::Runtime, + pub(crate) client: WorkerClient, + pub(crate) session: WorkerSession, +} + +pub(crate) fn unavailable() -> pyo3::PyErr { + PyRuntimeError::new_err("native worker session is closed or unavailable") +} + +impl WorkerOwner { + pub(crate) fn close(self) { + drop(self.session); + drop(self.client); + self.runtime + .shutdown_timeout(std::time::Duration::from_secs(1)); + } +} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/compile_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/compile_declarations.rs new file mode 100644 index 0000000000..de4e6710ad --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/compile_declarations.rs @@ -0,0 +1,25 @@ +//! Compile declarations. + +use crate::graph::models::ProjectSelection; +use crate::project_snapshot::_helpers::snapshot::snapshot_declarations; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use std::path::Path; + +/// Compile declarations. +#[inline] +pub fn compile_declarations( + py: Python<'_>, + input: SnapshotInput, + (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), + (inventory, declarations): (&OrchestrationInventory, Vec>), +) -> PyResult { + snapshot_declarations::compile_declarations( + py, + input, + (project_dir, graph, selected), + (inventory, declarations), + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/compile_project_roles.rs similarity index 80% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs rename to crates/sqlbuild-python-project/src/project_snapshot/main/compile_project_roles.rs index 895bfa9eb9..64f3cbcaa2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/compile_project_roles.rs @@ -1,20 +1,17 @@ -//! Project roles from the reviewed `roles.toml` file, their selectors resolved against this snapshot. - -use std::path::Path; +//! Compile project roles. +use crate::graph::models::ProjectSelection; +use crate::project_snapshot::_helpers::snapshot::snapshot_roles::ROLES_FILE; use pyo3::exceptions::PyValueError; use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_orchestration::authorization::main::compile_roles::compile_roles; use sqlbuild_orchestration::snapshots::models::SnapshotInput; - -use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use sqlbuild_python_base::native_host::main::write_console::write_console; +use std::path::Path; -/// The project-root file that defines project roles. -pub(crate) const ROLES_FILE: &str = "roles.toml"; - -pub(crate) fn compile_project_roles( +/// Compile project roles. +pub fn compile_project_roles( py: Python<'_>, mut input: SnapshotInput, (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/mod.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/mod.rs new file mode 100644 index 0000000000..ad14c41e07 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/mod.rs @@ -0,0 +1,7 @@ +pub mod compile_declarations; +pub mod compile_project_roles; +pub mod model_nodes; +pub mod python_declarations; +pub mod register_snapshot; +pub mod relative_path; +pub mod resource_nodes; diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/model_nodes.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/model_nodes.rs new file mode 100644 index 0000000000..45583b6cfe --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/model_nodes.rs @@ -0,0 +1,20 @@ +//! Each model's snapshot node, and the `columns` document of ranges kept out of node identity. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::NativeProjectView; +use crate::project_snapshot::_helpers::snapshot::snapshot_models; +use pyo3::prelude::{PyResult, Python}; +use serde_json::{Map, Value}; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; +use std::collections::BTreeMap; + +/// Each model's snapshot node, and the `columns` document of ranges kept out of node identity. +#[inline] +pub fn model_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + nodes: BTreeMap, +) -> PyResult<(BTreeMap, Map)> { + snapshot_models::model_nodes(py, project, owner, nodes) +} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/python_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/python_declarations.rs new file mode 100644 index 0000000000..468fc93243 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/python_declarations.rs @@ -0,0 +1,27 @@ +//! The calendars and sensors of the inventory's modules, imported by publication and by sensor evaluation. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; +use sqlbuild_discovery::tree::models::ProjectTree; +use sqlbuild_python_base::orchestration_reads::main::module_declarations::module_declarations; +use std::collections::BTreeSet; +use std::path::Path; + +/// The calendars and sensors of the inventory's modules, imported by publication and by sensor evaluation. +pub fn python_declarations<'py>( + py: Python<'py>, + project_dir: &Path, + inventory: &OrchestrationInventory, +) -> PyResult>> { + let tree = ProjectTree::new(project_dir); + let mut declarations: Vec> = Vec::new(); + let mut seen: BTreeSet = BTreeSet::new(); + for module in inventory.calendars.iter().chain(&inventory.sensors) { + for declaration in module_declarations(py, &tree.absolute(&module.path), project_dir)? { + if seen.insert(declaration.as_ptr() as usize) { + declarations.push(declaration); + } + } + } + Ok(declarations) +} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/register_snapshot.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/register_snapshot.rs new file mode 100644 index 0000000000..c699f58b14 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/register_snapshot.rs @@ -0,0 +1,11 @@ +//! Register the snapshot functions and classes. + +use crate::project_snapshot::models::NativeOrchestrationSnapshot; +use crate::project_snapshot::models::NativeOrchestrationWorker; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the snapshot functions and classes. +pub fn register_snapshot(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::() +} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/relative_path.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/relative_path.rs new file mode 100644 index 0000000000..8ca9b45a52 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/relative_path.rs @@ -0,0 +1,10 @@ +//! Relative path. + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_base::records::main::fields::path_posix::path_posix; + +/// Relative path. +pub fn relative_path(value: &Bound<'_, PyAny>) -> PyResult { + path_posix(&field(value, "relative_path")?) +} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/resource_nodes.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/resource_nodes.rs new file mode 100644 index 0000000000..e14e160f77 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/resource_nodes.rs @@ -0,0 +1,19 @@ +//! Resource nodes. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::models::NativeProjectView; +use crate::project_snapshot::_helpers::snapshot::snapshot_resources; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_orchestration::snapshots::models::SnapshotNode; +use std::collections::BTreeMap; + +/// Resource nodes. +#[inline] +pub fn resource_nodes( + py: Python<'_>, + project: &NativeProjectView, + owner: &NativeCompiledProject, + nodes: BTreeMap, +) -> PyResult> { + snapshot_resources::resource_nodes(py, project, owner, nodes) +} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/mod.rs b/crates/sqlbuild-python-project/src/project_snapshot/mod.rs new file mode 100644 index 0000000000..a111c35a51 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/mod.rs @@ -0,0 +1,6 @@ +//! The compiled-project snapshot written and read across processes: models, resources, declarations, roles and its worker. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python-project/src/project_snapshot/models.rs similarity index 62% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs rename to crates/sqlbuild-python-project/src/project_snapshot/models.rs index e1731445c6..d4dc746edd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/models.rs @@ -1,61 +1,148 @@ -//! Frozen selection, executor callbacks and completion belong to one verified worker build. - -use std::collections::{BTreeMap, BTreeSet, VecDeque}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; - -use pyo3::exceptions::{PyKeyboardInterrupt, PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +//! Public records of project snapshot. + +use crate::compiled_project::constants::NO_CHANGE_REASON; +use crate::compiled_project::constants::REPORT_ATTEMPTS; +use crate::compiled_project::constants::SKIP_ACTION; +use crate::compiled_project::constants::SQL_TEST_KEY_PREFIX; +use crate::compiled_project::constants::STORE_BUSY_STATUS; +use crate::project_snapshot::_helpers::snapshot::snapshot_execution::Completion; +use crate::project_snapshot::_helpers::snapshot::snapshot_execution::is_function; +use crate::project_snapshot::_helpers::snapshot::snapshot_execution::poisoned; +use crate::project_snapshot::_helpers::snapshot::snapshot_execution::severity; +use crate::project_snapshot::_helpers::snapshot::snapshot_worker::WorkerOwner; +use crate::project_snapshot::_helpers::snapshot::snapshot_worker::unavailable; +use crate::project_snapshot::types::NodeKeys; +use pyo3::exceptions::PyKeyboardInterrupt; +use pyo3::exceptions::PyRuntimeError; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyRef; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; use pyo3::types::PyStringMethods; -use sqlbuild_core::models::{CompiledResourceType, ResourceKind}; +use sqlbuild_core::models::ResourceKind; use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; use sqlbuild_host::models::SqlTestOutcome; use sqlbuild_host::types::ResultEnum; -use sqlbuild_orchestration::runs::models::{ - NodeOutcome, RunChange, RunCommand, RunExecution, RunOutcome, -}; - -use crate::bindings::_helpers::compiled_project::constants::{ - NO_CHANGE_REASON, SKIP_ACTION, SQL_TEST_KEY_PREFIX, -}; -use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; +use sqlbuild_orchestration::api::errors::ClientError; +use sqlbuild_orchestration::runs::models::NodeOutcome; +use sqlbuild_orchestration::runs::models::RunChange; +use sqlbuild_orchestration::runs::models::RunCommand; +use sqlbuild_orchestration::runs::models::RunExecution; +use sqlbuild_orchestration::runs::models::RunOutcome; +use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use sqlbuild_orchestration::snapshots::models::SnapshotRecord; +use sqlbuild_orchestration::worker::errors::WorkerError; +use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; +use sqlbuild_orchestration::worker::main::start_worker::start_worker; use sqlbuild_python_base::records::main::fields::field::field; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use uuid::Uuid; + +/// A data-only publication; this object never contains adapters, connections or user callables. +#[pyclass(frozen, module = "sqlbuild._native")] +#[derive(Debug)] +pub struct NativeOrchestrationSnapshot { + pub input: SnapshotInput, +} -/// Node keys by snapshot node key, such as the models each test checks. -pub(crate) type NodeKeys = BTreeMap>; +impl NativeOrchestrationSnapshot { + /// Native callers retain transport errors so they can report an uncertain acknowledgement. + pub fn publication( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + ) -> PyResult> { + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let result = runtime.block_on(publish_remote_snapshot(server_url, bearer, &self.input)); + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + Ok(result) + }) + } +} -/// A finished node's outcome, rows affected, error code and error message. -type Completion = (NodeOutcome, Option, (Option, Option)); +#[pymethods] +impl NativeOrchestrationSnapshot { + /// Serialize only at the public boundary; native transports can use the retained input. + pub(crate) fn to_json(&self, py: Python<'_>) -> PyResult { + py.detach(|| serde_json::to_string(&self.input)) + .map_err(|error| PyValueError::new_err(error.to_string())) + } + + /// Publish retained data without projecting it through Python, releasing the GIL during IO. + pub(crate) fn publish( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + ) -> PyResult { + let record = self + .publication(py, server_url, bearer)? + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + py.detach(|| { + serde_json::to_string(&record) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Retain the verified claim and its independent native heartbeat in the worker interpreter. + pub(crate) fn start_worker( + &self, + py: Python<'_>, + server_url: &str, + bearer: &str, + identity: (String, String), + ) -> PyResult { + NativeOrchestrationWorker::start(self, py, (server_url, bearer), identity) + } +} /// This transient owner records only acknowledged outcomes; durable lifecycle remains on the API. #[derive(Debug)] -pub(crate) struct WorkerBuild { - pub(crate) worker: Py, - execution: RunExecution, - completed: Mutex>, - checks: Mutex>>, +pub struct WorkerBuild { + pub worker: Py, + pub(crate) execution: RunExecution, + pub(crate) completed: Mutex>, + pub(crate) checks: Mutex>>, /// The snapshot's audits and SQL tests, each with the node keys it checks. - attached: BTreeMap>, + pub(crate) attached: BTreeMap>, /// Each SQL test with every node a failure of it blocks in a build: its expected, assertion and reference models and the functions it checks. - blocked: BTreeMap>, + pub(crate) blocked: BTreeMap>, /// How many of each function's planned tests have still to report; a function is built before its tests run. - untested: Mutex>, + pub(crate) untested: Mutex>, /// A built function's result, held until its tests report so a failing one can fail it instead. - held: Mutex>, + pub(crate) held: Mutex>, /// Each snapshot node's direct upstream node keys, for naming what a skipped node waited on. - upstream: BTreeMap>, + pub(crate) upstream: BTreeMap>, /// The failed node each finished node traces back to: itself when it failed, its cause when skipped. - causes: Mutex>, + pub(crate) causes: Mutex>, /// The worst attached outcome of each node, for commands that only run tests or audits. - checked: Mutex>, + pub(crate) checked: Mutex>, /// Whether the command returned, having run every check it planned, rather than being refused or raising. - returned: AtomicBool, + pub(crate) returned: AtomicBool, /// Why the worker refused the run before executing anything, reported as the run's status reason. - refusal: Mutex>, + pub(crate) refusal: Mutex>, } impl WorkerBuild { - pub(crate) fn new( + pub fn new( worker: &Bound<'_, NativeOrchestrationWorker>, (attached, blocked, upstream): (NodeKeys, NodeKeys, NodeKeys), ) -> PyResult> { @@ -80,26 +167,26 @@ impl WorkerBuild { Ok(state) } - pub(crate) fn command(&self) -> RunCommand { + pub fn command(&self) -> RunCommand { self.execution.claim.request.command } /// The server run this build executes. - pub(crate) fn run_id(&self) -> uuid::Uuid { + pub fn run_id(&self) -> uuid::Uuid { self.execution.claim.request.run_id } /// Test and audit runs execute only attached checks, so their nodes take the checks' outcome. - fn checks_only(&self) -> bool { + pub(crate) fn checks_only(&self) -> bool { matches!(self.command(), RunCommand::Test | RunCommand::Audit) } /// Record that the command returned instead of being refused or raising. - pub(crate) fn command_returned(&self) { + pub fn command_returned(&self) { self.returned.store(true, Ordering::SeqCst); } - pub(crate) fn has_work(&self) -> bool { + pub fn has_work(&self) -> bool { self.execution .claim .request @@ -109,12 +196,12 @@ impl WorkerBuild { } /// Record why the run is refused; the terminal report carries it as the run's status reason. - pub(crate) fn refuse(&self, reason: &str) -> PyResult<()> { + pub fn refuse(&self, reason: &str) -> PyResult<()> { *self.refusal.lock().map_err(|_| poisoned())? = Some(reason.to_owned()); Ok(()) } - pub(crate) fn check_cancelled(&self, py: Python<'_>) -> PyResult<()> { + pub fn check_cancelled(&self, py: Python<'_>) -> PyResult<()> { if self.worker.bind(py).get().cancel_requested(py) { return Err(PyKeyboardInterrupt::new_err("run cancellation requested")); } @@ -122,11 +209,7 @@ impl WorkerBuild { } /// Reject writers outside the frozen write set before any executor connection. - pub(crate) fn enforce_writers( - &self, - py: Python<'_>, - keys: &[String], - ) -> Result<(), pyo3::PyErr> { + pub fn enforce_writers(&self, py: Python<'_>, keys: &[String]) -> Result<(), pyo3::PyErr> { self.check_cancelled(py)?; match keys .iter() @@ -140,10 +223,7 @@ impl WorkerBuild { } /// Reject an implicitly expanded write set before any executor connection or schema change. - pub(crate) fn enforce_plan_writers( - &self, - pipeline: &Bound<'_, PyAny>, - ) -> Result<(), pyo3::PyErr> { + pub fn enforce_plan_writers(&self, pipeline: &Bound<'_, PyAny>) -> Result<(), pyo3::PyErr> { self.check_cancelled(pipeline.py())?; let plan = field(pipeline, "plan_output")?; self.count_planned_tests(&plan)?; @@ -182,7 +262,7 @@ impl WorkerBuild { } /// Count each function's tests this build plans, so a built function waits only for tests that will report. - fn count_planned_tests(&self, plan: &Bound<'_, PyAny>) -> PyResult<()> { + pub(crate) fn count_planned_tests(&self, plan: &Bound<'_, PyAny>) -> PyResult<()> { let mut untested = self.untested.lock().map_err(|_| poisoned())?; for entry in field(plan, "test_entries")?.try_iter()? { let test = format!("{SQL_TEST_KEY_PREFIX}{}", field(&entry?, "name")?.str()?); @@ -196,7 +276,7 @@ impl WorkerBuild { } /// Nodes `build --changed` leaves unbuilt are up to date, as a planner `no_change` skip is. - pub(crate) fn unchanged(&self, py: Python<'_>, keys: &[String]) -> PyResult<()> { + pub fn unchanged(&self, py: Python<'_>, keys: &[String]) -> PyResult<()> { for key in keys { self.complete(py, key, (NodeOutcome::Success, None, (None, None)))?; } @@ -204,7 +284,7 @@ impl WorkerBuild { } /// Nodes `build --changed` holds back wait for the job that refreshes them, with why. - pub(crate) fn held_back(&self, py: Python<'_>, nodes: &[(String, String)]) -> PyResult<()> { + pub fn held_back(&self, py: Python<'_>, nodes: &[(String, String)]) -> PyResult<()> { for (key, why) in nodes { self.complete( py, @@ -220,7 +300,7 @@ impl WorkerBuild { } /// A skip in a build names the failed node upstream of it; a test or audit run that raised skips what it never checked, with no cause. - fn complete( + pub(crate) fn complete( &self, py: Python<'_>, key: &str, @@ -278,7 +358,7 @@ impl WorkerBuild { Ok(()) } - pub(crate) fn start( + pub fn start( &self, py: Python<'_>, name: &str, @@ -294,7 +374,7 @@ impl WorkerBuild { } /// Report a selected node as started unless its outcome is final, such as a planner skip. - pub(crate) fn start_key(&self, py: Python<'_>, key: &str) -> PyResult<()> { + pub fn start_key(&self, py: Python<'_>, key: &str) -> PyResult<()> { self.check_cancelled(py)?; if self.execution.claim.request.selection.contains(key) && !self @@ -315,17 +395,12 @@ impl WorkerBuild { } /// Report a selected node's outcome when the command reports no executor result for it. - pub(crate) fn finish_key( - &self, - py: Python<'_>, - key: &str, - outcome: NodeOutcome, - ) -> PyResult<()> { + pub fn finish_key(&self, py: Python<'_>, key: &str, outcome: NodeOutcome) -> PyResult<()> { self.complete(py, key, (outcome, None, (None, None))) } /// A test or audit the snapshot records has started; a selected SQL test reports as a node. - pub(crate) fn start_attached(&self, py: Python<'_>, key: &str) -> PyResult<()> { + pub fn start_attached(&self, py: Python<'_>, key: &str) -> PyResult<()> { self.check_cancelled(py)?; let Some(nodes) = self.attached.get(key) else { return Ok(()); @@ -361,12 +436,7 @@ impl WorkerBuild { } /// A test or audit finished; a check the snapshot does not record is not reported. - pub(crate) fn finish_attached( - &self, - py: Python<'_>, - key: &str, - outcome: NodeOutcome, - ) -> PyResult<()> { + pub fn finish_attached(&self, py: Python<'_>, key: &str, outcome: NodeOutcome) -> PyResult<()> { let Some(nodes) = self.attached.get(key) else { return Ok(()); }; @@ -403,7 +473,7 @@ impl WorkerBuild { } /// The failed node nearest upstream of `key` in this run, or the cause a skipped one carries. - fn failed_upstream(&self, key: &str) -> PyResult> { + pub(crate) fn failed_upstream(&self, key: &str) -> PyResult> { let causes = self.causes.lock().map_err(|_| poisoned())?; let mut seen: BTreeSet<&str> = BTreeSet::from([key]); let mut pending: VecDeque<&str> = VecDeque::from([key]); @@ -421,7 +491,7 @@ impl WorkerBuild { } /// A selected SQL test completes itself; a failing one, found by its compiled name such as `parent [case]`, fails the nodes it blocks; each test that reports releases a function waiting on its tests. - fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { + pub(crate) fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { let py = result.py(); let outcome: String = field(result, "outcome")?.str()?.extract()?; let passed = SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass); @@ -455,7 +525,7 @@ impl WorkerBuild { } /// Count `test` as reported for each function it checks, and publish a held function once none of its tests is left. - fn release_tested(&self, py: Python<'_>, test: &str) -> PyResult<()> { + pub(crate) fn release_tested(&self, py: Python<'_>, test: &str) -> PyResult<()> { let mut released: Vec<(String, Completion)> = Vec::new(); { let mut untested = self.untested.lock().map_err(|_| poisoned())?; @@ -479,7 +549,12 @@ impl WorkerBuild { } /// Publish a node's result, or hold a built function's while its tests have still to report. - fn complete_or_hold(&self, py: Python<'_>, key: &str, completion: Completion) -> PyResult<()> { + pub(crate) fn complete_or_hold( + &self, + py: Python<'_>, + key: &str, + completion: Completion, + ) -> PyResult<()> { let waiting = !self.checks_only() && matches!(completion.0, NodeOutcome::Success | NodeOutcome::Warn) && self @@ -499,7 +574,7 @@ impl WorkerBuild { } /// Existing native progress callbacks publish selected-node evidence without Python relays. - pub(crate) fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { + pub fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { if key.starts_with(SQL_TEST_KEY_PREFIX) { return self.finished_test(result); } @@ -551,7 +626,7 @@ impl WorkerBuild { Ok(()) } - pub(crate) fn start_check(&self, py: Python<'_>, name: &str) -> PyResult<()> { + pub fn start_check(&self, py: Python<'_>, name: &str) -> PyResult<()> { self.check_cancelled(py)?; let key = format!("check:{name}"); self.worker.bind(py).get().change( @@ -569,7 +644,7 @@ impl WorkerBuild { } /// A completion without our actual-execution start callback is a blocked check, not a failure. - pub(crate) fn completed_check(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { + pub fn completed_check(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { let name: String = field(result, "node_name")?.extract()?; let key = format!("check:{name}"); let started = self @@ -596,7 +671,12 @@ impl WorkerBuild { self.finish_check(result.py(), &key, outcome) } - fn finish_check(&self, py: Python<'_>, key: &str, outcome: NodeOutcome) -> PyResult<()> { + pub(crate) fn finish_check( + &self, + py: Python<'_>, + key: &str, + outcome: NodeOutcome, + ) -> PyResult<()> { self.worker.bind(py).get().change( py, &format!("check-finished/{key}"), @@ -613,7 +693,7 @@ impl WorkerBuild { } /// Never call unseen nodes successful unless the executor returned; a refused command proves only its failures. - pub(crate) fn outcome(&self, py: Python<'_>, exit_code: i32) -> PyResult { + pub fn outcome(&self, py: Python<'_>, exit_code: i32) -> PyResult { let cancelled = self.worker.bind(py).get().refresh_cancellation(py)?; let checks_only = self.checks_only(); if checks_only && !cancelled { @@ -682,7 +762,7 @@ impl WorkerBuild { } /// Terminal publication follows executor teardown and the invocation's bounded Recall drain. - pub(crate) fn finish(&self, py: Python<'_>, exit_code: i32) -> PyResult { + pub fn finish(&self, py: Python<'_>, exit_code: i32) -> PyResult { let outcome = self.outcome(py, exit_code)?; let unfinished: Vec = self .checks @@ -722,29 +802,190 @@ impl WorkerBuild { } } -/// A claimed run's worker build and the project inputs it verified against the frozen snapshot. -pub(crate) type VerifiedRun<'a, 'py> = (Arc, &'a Bound<'py, PyAny>); +/// Closing this owner stops and joins heartbeat IO but does not claim warehouse work has stopped. +#[pyclass(frozen, module = "sqlbuild._native")] +#[derive(Debug)] +pub struct NativeOrchestrationWorker { + pub(crate) owner: Mutex>, +} -/// Failure outranks cancellation, then warnings, then success. -fn severity(outcome: NodeOutcome) -> u8 { - match outcome { - NodeOutcome::Skipped => 0, - NodeOutcome::Success => 1, - NodeOutcome::Warn => 2, - NodeOutcome::Cancelled => 3, - NodeOutcome::Failed => 4, +impl NativeOrchestrationWorker { + /// Run `work` on the open owner without holding the GIL; a closed worker is unavailable. + pub(crate) fn with_owner( + &self, + py: Python<'_>, + work: impl FnOnce(&WorkerOwner) -> PyResult + Send, + ) -> PyResult { + py.detach(|| { + let owner = self.owner.lock().map_err(|_| unavailable())?; + work(owner.as_ref().ok_or_else(unavailable)?) + }) + } + + /// SIGTERM may arrive before the periodic heartbeat observes the durable cancellation fact. + pub fn refresh_cancellation(&self, py: Python<'_>) -> PyResult { + self.with_owner(py, |owner| { + if owner.session.cancel_requested() { + return Ok(true); + } + owner + .runtime + .block_on(owner.client.heartbeat()) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Native execution reads the retained request, never caller-provided selection or settings. + pub fn execution( + &self, + py: Python<'_>, + ) -> PyResult { + self.with_owner(py, |owner| Ok(owner.session.execution.clone())) + } + + /// A bounded retry keeps the event identity across an uncertain acknowledgement or busy store. + pub fn change(&self, py: Python<'_>, key: &str, change: RunChange) -> PyResult<()> { + self.with_owner(py, |owner| { + let event_id = Uuid::new_v5(&owner.session.execution.claim.claim_id, key.as_bytes()); + owner.runtime.block_on(async { + let mut attempts = 0; + loop { + attempts += 1; + match owner.client.report(event_id, change.clone()).await { + Ok(_) => return Ok(()), + Err(error) + if attempts < REPORT_ATTEMPTS + && match &error { + WorkerError::Transport(_) => true, + WorkerError::Rejected(status) => { + status.as_u16() == STORE_BUSY_STATUS + } + _ => false, + } => + { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + Err(error) => return Err(PyRuntimeError::new_err(error.to_string())), + } + } + }) + }) + } + + /// Verify this retained baseline before the caller applies run settings or opens a warehouse. + pub fn start( + snapshot: &NativeOrchestrationSnapshot, + py: Python<'_>, + (server_url, bearer): (&str, &str), + (run_id, claim_id): (String, String), + ) -> PyResult { + let identity = ( + Uuid::parse_str(&run_id).map_err(|error| PyValueError::new_err(error.to_string()))?, + Uuid::parse_str(&claim_id).map_err(|error| PyValueError::new_err(error.to_string()))?, + ); + py.detach(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + let connected = runtime.block_on(async { + let client = connect_worker(server_url, bearer, identity)?; + let session = start_worker(&client, &snapshot.input).await?; + Ok::<_, sqlbuild_orchestration::worker::errors::WorkerError>((client, session)) + }); + match connected { + Ok((client, session)) => Ok(NativeOrchestrationWorker { + owner: Mutex::new(Some(WorkerOwner { + runtime, + client, + session, + })), + }), + Err(error) => { + runtime.shutdown_timeout(std::time::Duration::from_secs(1)); + Err(PyRuntimeError::new_err(error.to_string())) + } + } + }) } } -/// Whether a snapshot node key names a scalar or table function. -fn is_function(key: &str) -> bool { - key.split_once(':').is_some_and(|(kind, _)| { - [CompiledResourceType::Udf, CompiledResourceType::TableFn] - .iter() - .any(|function| function.as_str() == kind) - }) +#[pymethods] +impl NativeOrchestrationWorker { + /// The frozen request supplies selection and settings only after baseline verification. + pub(crate) fn execution_json(&self, py: Python<'_>) -> PyResult { + self.with_owner(py, |owner| { + serde_json::to_string(&owner.session.execution) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + /// Cancellation remains sticky; a closed worker cannot authorize further work. + pub fn cancel_requested(&self, py: Python<'_>) -> bool { + py.detach(|| { + self.owner.lock().map_or(true, |owner| { + owner + .as_ref() + .is_none_or(|owner| owner.session.cancel_requested()) + }) + }) + } + + /// Retry an uncertain report with the identical event ID and change, never a new identity. + pub(crate) fn report( + &self, + py: Python<'_>, + event_id: &str, + change_json: &str, + ) -> PyResult { + let event_id = + Uuid::parse_str(event_id).map_err(|error| PyValueError::new_err(error.to_string()))?; + let change: RunChange = serde_json::from_str(change_json) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + self.with_owner(py, |owner| { + owner + .runtime + .block_on(owner.client.report(event_id, change)) + .map_err(|error| PyRuntimeError::new_err(error.to_string())) + }) + } + + pub(crate) fn __enter__(slf: PyRef<'_, Self>) -> PyResult> { + if slf.owner.lock().map_err(|_| unavailable())?.is_none() { + return Err(unavailable()); + } + Ok(slf) + } + + pub(crate) fn __exit__( + &self, + py: Python<'_>, + _kind: &pyo3::Bound<'_, pyo3::PyAny>, + _value: &pyo3::Bound<'_, pyo3::PyAny>, + _traceback: &pyo3::Bound<'_, pyo3::PyAny>, + ) -> PyResult<()> { + self.close(py) + } + + /// Join the heartbeat without publishing a terminal outcome or releasing physical claims. + pub(crate) fn close(&self, py: Python<'_>) -> PyResult<()> { + py.detach(|| { + let mut owner = self.owner.lock().map_err(|_| unavailable())?; + if let Some(active) = owner.take() { + active.close(); + } + Ok(()) + }) + } } -fn poisoned() -> pyo3::PyErr { - PyRuntimeError::new_err("worker build outcome lock is unavailable") +impl Drop for NativeOrchestrationWorker { + /// GC can hold the GIL; heartbeat teardown joins native IO and never needs Python. + fn drop(&mut self) { + if let Ok(owner) = self.owner.get_mut() + && let Some(owner) = owner.take() + { + owner.close(); + } + } } diff --git a/crates/sqlbuild-python-project/src/project_snapshot/types.rs b/crates/sqlbuild-python-project/src/project_snapshot/types.rs new file mode 100644 index 0000000000..daee71c2f8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/types.rs @@ -0,0 +1,13 @@ +//! Public traits and type aliases of project snapshot. + +use crate::project_snapshot::models::WorkerBuild; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use std::collections::BTreeMap; +use std::sync::Arc; + +/// Node keys by snapshot node key, such as the models each test checks. +pub type NodeKeys = BTreeMap>; + +/// A claimed run's worker build and the project inputs it verified against the frozen snapshot. +pub type VerifiedRun<'a, 'py> = (Arc, &'a Bound<'py, PyAny>); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs b/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs similarity index 81% rename from crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs rename to crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs index 693a47d4e9..d289897091 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs +++ b/crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs @@ -1,21 +1,17 @@ //! Compile's explicit-reference checks over plain project facts, run natively. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use crate::references::models::GateRow; +use crate::references::models::HookRow; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use std::collections::{BTreeMap, BTreeSet}; use pyo3::exceptions::{PyOSError, PyTypeError}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PySet, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::explicit_references::errors::ReferenceFailure; use sqlbuild_analysis::explicit_references::main::check_explicit_references::check_explicit_references; -use sqlbuild_analysis::explicit_references::main::execution_gate_edges::execution_gate_edges; -use sqlbuild_analysis::explicit_references::main::old_name_message::old_name_message; -use sqlbuild_analysis::explicit_references::main::relation_remedy::relation_remedy; use sqlbuild_analysis::explicit_references::models::{ - ExecutionGateEdges, ExecutionGateRequest, GateEdge, HookEdge, HookType, LiteralSql, - ReferenceReport, ReferenceRequest, SqlOwner, + GateEdge, HookEdge, HookType, LiteralSql, ReferenceReport, ReferenceRequest, SqlOwner, }; use sqlbuild_analysis::explicit_references::types::ResourceKey; use sqlbuild_config::project::main::explicit_references_help::explicit_references_help as references_help; @@ -24,8 +20,8 @@ use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_pysource::literal_sql::main::literal_sql_calls::literal_sql_calls; use sqlbuild_pysource::literal_sql::models::FunctionSource; -use crate::bindings::_helpers::graph::execution_gates::model_python_hook_names; -use crate::bindings::_helpers::references::reference_facts::{ProjectFacts, project_facts}; +use crate::graph::main::nodes::model_python_hook_names::model_python_hook_names; +use crate::references::_helpers::reference_facts::{ProjectFacts, project_facts}; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -34,7 +30,6 @@ use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; @@ -393,74 +388,7 @@ fn python_literal_sql_calls( .collect() } -/// How an owner of `owner_kind` should read `target` instead of naming its relation in SQL. -#[pyfunction] -#[pyo3(signature = (*, owner_kind, r#ref, upstream_loader_by_source = None))] -pub(crate) fn hard_coded_relation_remedy( - owner_kind: &str, - r#ref: &Bound<'_, PyAny>, - upstream_loader_by_source: Option<&Bound<'_, PyAny>>, -) -> PyResult { - let target: ResourceKey = ( - native_field(r#ref, "kind")?.getattr("value")?.extract()?, - native_field(r#ref, "name")?.extract()?, - ); - let mut upstream_loaders: Vec<(String, String)> = Vec::new(); - if let Some(loaders) = upstream_loader_by_source.filter(|loaders| !loaders.is_none()) { - for item in mapping_items(loaders)?.try_iter()? { - upstream_loaders.push(item?.extract()?); - } - } - Ok(relation_remedy(owner_kind, &target, &upstream_loaders)) -} - -/// The P008 message for SQL naming a relation kept only as a compatibility view. -#[pyfunction] -#[pyo3(signature = (*, owner_label, written, method, model_name))] -pub(crate) fn old_name_reference_message( - owner_label: &str, - written: &str, - method: &str, - model_name: &str, -) -> String { - old_name_message(owner_label, written, method, model_name) -} - -/// One attached-audit ordering edge: the audit, its target, and the gated and read resources. -pub(crate) struct GateRow { - pub(crate) audit_name: String, - pub(crate) target: ResourceKey, - pub(crate) gated: ResourceKey, - pub(crate) read: ResourceKey, -} - -/// One hook-read ordering edge: the hook, the gated and read resources, and its type. -pub(crate) struct HookRow { - pub(crate) hook_name: String, - pub(crate) gated: ResourceKey, - pub(crate) read: ResourceKey, - /// The `HookReadType` value. - pub(crate) hook_type: &'static str, -} - -/// The attached-audit gate edges and hook-read edges of one project, in Python's order. -pub(crate) fn native_execution_gate_edges( - project: &Bound<'_, PyAny>, -) -> PyResult<(Vec, Vec)> { - let facts: ProjectFacts = project_facts(project, false)?; - let edges: ExecutionGateEdges = execution_gate_edges(&ExecutionGateRequest { - resources: facts.resources, - models: facts.models, - audits: facts.audits, - hooks: facts.hooks, - }); - Ok(( - edges.audit_gates.into_iter().map(gate_row).collect(), - edges.hook_reads.into_iter().map(hook_row).collect(), - )) -} - -fn gate_row(edge: GateEdge) -> GateRow { +pub(crate) fn gate_row(edge: GateEdge) -> GateRow { GateRow { audit_name: edge.audit_name, target: edge.target, @@ -469,7 +397,7 @@ fn gate_row(edge: GateEdge) -> GateRow { } } -fn hook_row(edge: HookEdge) -> HookRow { +pub(crate) fn hook_row(edge: HookEdge) -> HookRow { let hook_type: &'static str = match edge.hook_type { HookType::Python => "Python", HookType::Sql => "SQL", @@ -482,10 +410,3 @@ fn hook_row(edge: HookEdge) -> HookRow { hook_type, } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(explicit_reference_report, module)?)?; - module.add_function(wrap_pyfunction!(hard_coded_relation_remedy, module)?)?; - module.add_function(wrap_pyfunction!(old_name_reference_message, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/references/_helpers/mod.rs b/crates/sqlbuild-python-project/src/references/_helpers/mod.rs new file mode 100644 index 0000000000..2bc3b431b5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod explicit_references; +pub(crate) mod reference_facts; +pub(crate) mod relations; +pub(crate) mod source_relations; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/reference_facts.rs b/crates/sqlbuild-python-project/src/references/_helpers/reference_facts.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/references/reference_facts.rs rename to crates/sqlbuild-python-project/src/references/_helpers/reference_facts.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs b/crates/sqlbuild-python-project/src/references/_helpers/relations.rs similarity index 59% rename from crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs rename to crates/sqlbuild-python-project/src/references/_helpers/relations.rs index 969f55318f..16a3174444 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs +++ b/crates/sqlbuild-python-project/src/references/_helpers/relations.rs @@ -1,16 +1,14 @@ //! Relation names in SQL and the project relations they match, for hard-coded name checks. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::text::main::python_casefold::python_casefold; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::analysis::relation_names::{ - RelationRow, extract_sql_relation_names, -}; +use crate::analysis::main::extract_sql_relation_names::extract_sql_relation_names; +use crate::analysis::types::RelationRow; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; const RUNTIME_RELATION_PREFIX: &str = "SELECT 1 FROM "; @@ -40,7 +38,7 @@ fn relation_name<'py>( /// Relations named in `sql` and temporary tables it creates, or None if unparseable. #[pyfunction] #[pyo3(signature = (*, sql, dialect))] -fn extract_relation_names<'py>( +pub(crate) fn extract_relation_names<'py>( py: Python<'py>, sql: &str, dialect: Option<&str>, @@ -62,50 +60,17 @@ fn extract_relation_names<'py>( } /// Whether two optional qualifier parts can name the same relation. -fn parts_compatible(left: Option<&str>, right: Option<&str>) -> bool { +pub(crate) fn parts_compatible(left: Option<&str>, right: Option<&str>) -> bool { match (left, right) { (Some(left), Some(right)) => python_casefold(left) == python_casefold(right), _ => true, } } -/// The project relation `relation` names, resolving missing qualifiers from the defaults. -#[pyfunction] -#[pyo3(signature = (*, index, relation, default_database = None, default_schema = None))] -pub(crate) fn match_project_relation<'py>( - index: &Bound<'py, PyAny>, - relation: &Bound<'py, PyAny>, - default_database: Option, - default_schema: Option, -) -> PyResult>> { - let name: String = python_casefold(&native_field(relation, "name")?.extract::()?); - let schema: Option = native_field(relation, "schema")? - .extract::>()? - .or(default_schema); - let database: Option = native_field(relation, "database")? - .extract::>()? - .or(default_database); - for candidate in native_field(index, "relations")?.try_iter()? { - let candidate: Bound<'py, PyAny> = candidate?; - let target: Bound<'py, PyAny> = native_field(&candidate, "relation")?; - if python_casefold(&native_field(&target, "name")?.extract::()?) != name { - continue; - } - let target_schema: Option = native_field(&target, "schema")?.extract()?; - let target_database: Option = native_field(&target, "database")?.extract()?; - if parts_compatible(schema.as_deref(), target_schema.as_deref()) - && parts_compatible(database.as_deref(), target_database.as_deref()) - { - return Ok(Some(candidate)); - } - } - Ok(None) -} - /// Project relations from adapter-rendered runtime relation names, deduplicated in order. #[pyfunction] #[pyo3(signature = (*, relations, dialect))] -fn runtime_project_relations<'py>( +pub(crate) fn runtime_project_relations<'py>( py: Python<'py>, relations: &Bound<'py, PyAny>, dialect: Option<&str>, @@ -135,10 +100,3 @@ fn runtime_project_relations<'py>( &[("relations", PyTuple::new(py, resolved.keys())?.into_any())], ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(extract_relation_names, module)?)?; - module.add_function(wrap_pyfunction!(match_project_relation, module)?)?; - module.add_function(wrap_pyfunction!(runtime_project_relations, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs b/crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs new file mode 100644 index 0000000000..0e2fe20777 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs @@ -0,0 +1,62 @@ +//! A declared source rendered as the SQL table factor that reads it. + +use sqlbuild_host::entries::models::AdapterHost; + +use pyo3::prelude::{PyErr, Python}; +use pyo3::types::PyString; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; + +/// The SQL analysis dialect whose engines read replacing tables with `FINAL`. +pub(crate) const FINAL_READ_DIALECT: &str = "clickhouse"; +/// Error code for a `final` source on another adapter. +const FINAL_UNSUPPORTED_CODE: &str = "A010"; +/// Guidance for a `final` source on another adapter. +const FINAL_UNSUPPORTED_HELP: &str = + "Remove 'final' from the source; only ClickHouse reads sources with FINAL."; + +/// Leading words of a source expression that is a query rather than a relation. +const QUERY_PREFIXES: [&str; 3] = ["select", "with", "values"]; + +/// A source expression as a table factor: queries parenthesized, relations as written. +pub(crate) fn expression_relation(expression: &str) -> String { + let trimmed: &str = expression.trim(); + let stripped: &str = trimmed.strip_suffix(';').unwrap_or(trimmed).trim(); + if stripped.starts_with('(') { + return stripped.to_owned(); + } + let lowered: String = stripped.to_lowercase(); + if QUERY_PREFIXES + .iter() + .any(|prefix| lowered.starts_with(prefix)) + { + return format!("({stripped})"); + } + stripped.to_owned() +} + +/// The coded error for a `final` source on an adapter that cannot read with `FINAL`. +pub(crate) fn final_unsupported(py: Python<'_>, adapter_name: &str) -> PyErr { + match host_entry(py, AdapterHost::AdapterUserError).and_then(|class| { + call_with( + &class, + &[ + ( + "message", + PyString::new( + py, + &format!( + "adapter '{adapter_name}' does not support the source option 'final'" + ), + ) + .into_any(), + ), + ("code", PyString::new(py, FINAL_UNSUPPORTED_CODE).into_any()), + ("help", PyString::new(py, FINAL_UNSUPPORTED_HELP).into_any()), + ], + ) + }) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} diff --git a/crates/sqlbuild-python-project/src/references/main/explicit_reference_report.rs b/crates/sqlbuild-python-project/src/references/main/explicit_reference_report.rs new file mode 100644 index 0000000000..13df509e75 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/explicit_reference_report.rs @@ -0,0 +1,15 @@ +//! Return P005, P007 and P008 errors and unmatched Python-SQL relations; raise P001. + +use crate::references::_helpers::explicit_references; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; + +/// Return P005, P007 and P008 errors and unmatched Python-SQL relations; raise P001. +#[inline] +pub fn explicit_reference_report<'py>( + py: Python<'py>, + project: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, + enforce_explicit: bool, +) -> PyResult> { + explicit_references::explicit_reference_report(py, project, discovered_inputs, enforce_explicit) +} diff --git a/crates/sqlbuild-python-project/src/references/main/hard_coded_relation_remedy.rs b/crates/sqlbuild-python-project/src/references/main/hard_coded_relation_remedy.rs new file mode 100644 index 0000000000..ea9eef60f1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/hard_coded_relation_remedy.rs @@ -0,0 +1,29 @@ +//! How an owner of `owner_kind` should read `target` instead of naming its relation in SQL. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_analysis::explicit_references::main::relation_remedy::relation_remedy; +use sqlbuild_analysis::explicit_references::types::ResourceKey; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; + +/// How an owner of `owner_kind` should read `target` instead of naming its relation in SQL. +#[pyfunction] +#[pyo3(signature = (*, owner_kind, r#ref, upstream_loader_by_source = None))] +pub fn hard_coded_relation_remedy( + owner_kind: &str, + r#ref: &Bound<'_, PyAny>, + upstream_loader_by_source: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let target: ResourceKey = ( + native_field(r#ref, "kind")?.getattr("value")?.extract()?, + native_field(r#ref, "name")?.extract()?, + ); + let mut upstream_loaders: Vec<(String, String)> = Vec::new(); + if let Some(loaders) = upstream_loader_by_source.filter(|loaders| !loaders.is_none()) { + for item in mapping_items(loaders)?.try_iter()? { + upstream_loaders.push(item?.extract()?); + } + } + Ok(relation_remedy(owner_kind, &target, &upstream_loaders)) +} diff --git a/crates/sqlbuild-python-project/src/references/main/match_project_relation.rs b/crates/sqlbuild-python-project/src/references/main/match_project_relation.rs new file mode 100644 index 0000000000..d73d4e3b92 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/match_project_relation.rs @@ -0,0 +1,40 @@ +//! The project relation `relation` names, resolving missing qualifiers from the defaults. + +use crate::references::_helpers::relations::parts_compatible; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The project relation `relation` names, resolving missing qualifiers from the defaults. +#[pyfunction] +#[pyo3(signature = (*, index, relation, default_database = None, default_schema = None))] +pub fn match_project_relation<'py>( + index: &Bound<'py, PyAny>, + relation: &Bound<'py, PyAny>, + default_database: Option, + default_schema: Option, +) -> PyResult>> { + let name: String = python_casefold(&native_field(relation, "name")?.extract::()?); + let schema: Option = native_field(relation, "schema")? + .extract::>()? + .or(default_schema); + let database: Option = native_field(relation, "database")? + .extract::>()? + .or(default_database); + for candidate in native_field(index, "relations")?.try_iter()? { + let candidate: Bound<'py, PyAny> = candidate?; + let target: Bound<'py, PyAny> = native_field(&candidate, "relation")?; + if python_casefold(&native_field(&target, "name")?.extract::()?) != name { + continue; + } + let target_schema: Option = native_field(&target, "schema")?.extract()?; + let target_database: Option = native_field(&target, "database")?.extract()?; + if parts_compatible(schema.as_deref(), target_schema.as_deref()) + && parts_compatible(database.as_deref(), target_database.as_deref()) + { + return Ok(Some(candidate)); + } + } + Ok(None) +} diff --git a/crates/sqlbuild-python-project/src/references/main/mod.rs b/crates/sqlbuild-python-project/src/references/main/mod.rs new file mode 100644 index 0000000000..24512ac147 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/mod.rs @@ -0,0 +1,10 @@ +pub mod explicit_reference_report; +pub mod hard_coded_relation_remedy; +pub mod match_project_relation; +pub mod native_execution_gate_edges; +pub mod old_name_reference_message; +pub mod register_explicit_references; +pub mod register_relations; +pub mod register_source_relations; +pub mod render_source_read; +pub mod render_source_relation; diff --git a/crates/sqlbuild-python-project/src/references/main/native_execution_gate_edges.rs b/crates/sqlbuild-python-project/src/references/main/native_execution_gate_edges.rs new file mode 100644 index 0000000000..d3e025b284 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/native_execution_gate_edges.rs @@ -0,0 +1,27 @@ +//! The attached-audit gate edges and hook-read edges of one project, in Python's order. + +use crate::references::_helpers::explicit_references::gate_row; +use crate::references::_helpers::explicit_references::hook_row; +use crate::references::_helpers::reference_facts::{ProjectFacts, project_facts}; +use crate::references::models::GateRow; +use crate::references::models::HookRow; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_analysis::explicit_references::main::execution_gate_edges::execution_gate_edges; +use sqlbuild_analysis::explicit_references::models::{ExecutionGateEdges, ExecutionGateRequest}; + +/// The attached-audit gate edges and hook-read edges of one project, in Python's order. +pub fn native_execution_gate_edges( + project: &Bound<'_, PyAny>, +) -> PyResult<(Vec, Vec)> { + let facts: ProjectFacts = project_facts(project, false)?; + let edges: ExecutionGateEdges = execution_gate_edges(&ExecutionGateRequest { + resources: facts.resources, + models: facts.models, + audits: facts.audits, + hooks: facts.hooks, + }); + Ok(( + edges.audit_gates.into_iter().map(gate_row).collect(), + edges.hook_reads.into_iter().map(hook_row).collect(), + )) +} diff --git a/crates/sqlbuild-python-project/src/references/main/old_name_reference_message.rs b/crates/sqlbuild-python-project/src/references/main/old_name_reference_message.rs new file mode 100644 index 0000000000..b95e474fd5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/old_name_reference_message.rs @@ -0,0 +1,16 @@ +//! The P008 message for SQL naming a relation kept only as a compatibility view. + +use pyo3::pyfunction; +use sqlbuild_analysis::explicit_references::main::old_name_message::old_name_message; + +/// The P008 message for SQL naming a relation kept only as a compatibility view. +#[pyfunction] +#[pyo3(signature = (*, owner_label, written, method, model_name))] +pub fn old_name_reference_message( + owner_label: &str, + written: &str, + method: &str, + model_name: &str, +) -> String { + old_name_message(owner_label, written, method, model_name) +} diff --git a/crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs b/crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs new file mode 100644 index 0000000000..429725cf14 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs @@ -0,0 +1,15 @@ +//! Register the explicit references functions and classes. + +use crate::references::_helpers::explicit_references::explicit_reference_report; +use crate::references::main::hard_coded_relation_remedy::hard_coded_relation_remedy; +use crate::references::main::old_name_reference_message::old_name_reference_message; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the explicit references functions and classes. +pub fn register_explicit_references(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(explicit_reference_report, module)?)?; + module.add_function(wrap_pyfunction!(hard_coded_relation_remedy, module)?)?; + module.add_function(wrap_pyfunction!(old_name_reference_message, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/references/main/register_relations.rs b/crates/sqlbuild-python-project/src/references/main/register_relations.rs new file mode 100644 index 0000000000..ec601f0f32 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/register_relations.rs @@ -0,0 +1,15 @@ +//! Register the relations functions and classes. + +use crate::references::_helpers::relations::extract_relation_names; +use crate::references::_helpers::relations::runtime_project_relations; +use crate::references::main::match_project_relation::match_project_relation; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the relations functions and classes. +pub fn register_relations(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(extract_relation_names, module)?)?; + module.add_function(wrap_pyfunction!(match_project_relation, module)?)?; + module.add_function(wrap_pyfunction!(runtime_project_relations, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/references/main/register_source_relations.rs b/crates/sqlbuild-python-project/src/references/main/register_source_relations.rs new file mode 100644 index 0000000000..8b7aeb064b --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/register_source_relations.rs @@ -0,0 +1,11 @@ +//! Register the source relations functions and classes. + +use crate::references::main::render_source_relation::render_source_relation; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the source relations functions and classes. +pub fn register_source_relations(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_source_relation, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/references/main/render_source_read.rs b/crates/sqlbuild-python-project/src/references/main/render_source_read.rs new file mode 100644 index 0000000000..a3269c5408 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/render_source_read.rs @@ -0,0 +1,26 @@ +//! A source as the table factor that reads it, deduplicated with `FINAL` when `final`. + +use crate::references::_helpers::source_relations::FINAL_READ_DIALECT; +use crate::references::_helpers::source_relations::final_unsupported; +use crate::references::main::render_source_relation::render_source_relation; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A source as the table factor that reads it, deduplicated with `FINAL` when `final`. +pub fn render_source_read<'py>( + entry: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult { + let relation: String = render_source_relation(entry, Some(adapter))?; + if !native_field(entry, "final")?.is_truthy()? { + return Ok(relation); + } + let dialect: Option = native_field(adapter, "sql_analysis_dialect_name")?.extract()?; + if dialect.as_deref() != Some(FINAL_READ_DIALECT) { + return Err(final_unsupported( + entry.py(), + &native_field(adapter, "adapter_name")?.str()?.to_string(), + )); + } + Ok(format!("(SELECT * FROM {relation} FINAL)")) +} diff --git a/crates/sqlbuild-python-project/src/references/main/render_source_relation.rs b/crates/sqlbuild-python-project/src/references/main/render_source_relation.rs new file mode 100644 index 0000000000..2c3dee7663 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/main/render_source_relation.rs @@ -0,0 +1,54 @@ +//! Render a source as a SQL table factor, through the adapter when one is given. + +use crate::references::_helpers::source_relations::expression_relation; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Render a source as a SQL table factor, through the adapter when one is given. +#[pyfunction] +#[pyo3(signature = (*, entry, adapter = None))] +pub fn render_source_relation<'py>( + entry: &Bound<'py, PyAny>, + adapter: Option<&Bound<'py, PyAny>>, +) -> PyResult { + let adapter: Option<&Bound<'py, PyAny>> = adapter.filter(|adapter| !adapter.is_none()); + let expression: Option = native_field(entry, "expression")?.extract()?; + if let Some(expression) = expression { + return match adapter { + Some(adapter) => { + let keywords: Bound<'py, PyDict> = PyDict::new(entry.py()); + keywords.set_item("expression", expression)?; + adapter + .call_method("render_source_expression_relation", (), Some(&keywords))? + .extract() + } + None => Ok(expression_relation(&expression)), + }; + } + let table: Option = native_field(entry, "table")?.extract()?; + let table: String = match table { + Some(table) => table, + None => native_field(entry, "name")?.extract()?, + }; + let database: Option = native_field(entry, "database")?.extract()?; + let schema: Option = native_field(entry, "schema")?.extract()?; + if let Some(adapter) = adapter { + let keywords: Bound<'py, PyDict> = PyDict::new(entry.py()); + keywords.set_item("database", &database)?; + keywords.set_item("schema", &schema)?; + keywords.set_item("name", &table)?; + let rendered: Option = adapter + .call_method("render_qualified_name", (), Some(&keywords))? + .extract()?; + if let Some(rendered) = rendered { + return Ok(rendered); + } + } + Ok([database, schema, Some(table)] + .into_iter() + .flatten() + .collect::>() + .join(".")) +} diff --git a/crates/sqlbuild-python-project/src/references/mod.rs b/crates/sqlbuild-python-project/src/references/mod.rs new file mode 100644 index 0000000000..f89201734f --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/mod.rs @@ -0,0 +1,5 @@ +//! References between resources: explicit references, relations, source relations and their facts. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-project/src/references/models.rs b/crates/sqlbuild-python-project/src/references/models.rs new file mode 100644 index 0000000000..59b61e152b --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/models.rs @@ -0,0 +1,22 @@ +//! Public records of references. + +use sqlbuild_analysis::explicit_references::types::ResourceKey; + +/// One attached-audit ordering edge: the audit, its target, and the gated and read resources. +#[derive(Debug)] +pub struct GateRow { + pub audit_name: String, + pub target: ResourceKey, + pub gated: ResourceKey, + pub read: ResourceKey, +} + +/// One hook-read ordering edge: the hook, the gated and read resources, and its type. +#[derive(Debug)] +pub struct HookRow { + pub hook_name: String, + pub gated: ResourceKey, + pub read: ResourceKey, + /// The `HookReadType` value. + pub hook_type: &'static str, +} diff --git a/crates/sqlbuild-python-project/src/relation_naming/_helpers/fitting.rs b/crates/sqlbuild-python-project/src/relation_naming/_helpers/fitting.rs new file mode 100644 index 0000000000..1fd704bbf6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/relation_naming/_helpers/fitting.rs @@ -0,0 +1,38 @@ +//! Generated relation names fitted within identifier limits, raising `SharedInputError`. + +use crate::relation_naming::main::fitted_logical_name::fitted_logical_name; +use crate::relation_naming::main::shared_error::shared_error; +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_analysis::identifier_fitting::main::fit_auxiliary_relation_name::fit_auxiliary_relation_name as fit_auxiliary; + +use sqlbuild_python_base::boundary::models::Request; + +/// A readable logical component fitted after a fixed prefix, with a hash suffix when shortened. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn fit_artifact_logical_name(options: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("fit_artifact_logical_name", options)?; + fitted_logical_name( + request.py(), + ( + &request.get("logical_name")?.extract::()?, + &request.get("fixed_prefix")?.extract::()?, + ), + request.get("identifier_limit")?.extract()?, + &request.get("artifact_label")?.extract::()?, + ) +} + +/// `base_name + suffix` fitted within the limit, keeping the suffix and a base hash. +#[pyfunction] +#[pyo3(signature = (*, base_name, suffix, identifier_limit))] +pub(crate) fn fit_auxiliary_relation_name( + py: Python<'_>, + base_name: &str, + suffix: &str, + identifier_limit: i64, +) -> PyResult { + fit_auxiliary(base_name, suffix, identifier_limit).map_err(|message| shared_error(py, message)) +} diff --git a/crates/sqlbuild-python-project/src/relation_naming/_helpers/mod.rs b/crates/sqlbuild-python-project/src/relation_naming/_helpers/mod.rs new file mode 100644 index 0000000000..79e24dc7c0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/relation_naming/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod fitting; diff --git a/crates/sqlbuild-python-project/src/relation_naming/main/fitted_logical_name.rs b/crates/sqlbuild-python-project/src/relation_naming/main/fitted_logical_name.rs new file mode 100644 index 0000000000..cd82d91552 --- /dev/null +++ b/crates/sqlbuild-python-project/src/relation_naming/main/fitted_logical_name.rs @@ -0,0 +1,16 @@ +//! `fit_artifact_logical_name`, raising `SharedInputError` when the name cannot fit. + +use crate::relation_naming::main::shared_error::shared_error; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::identifier_fitting::main::fit_artifact_logical_name::fit_artifact_logical_name as fit_logical; + +/// `fit_artifact_logical_name`, raising `SharedInputError` when the name cannot fit. +pub fn fitted_logical_name( + py: Python<'_>, + names: (&str, &str), + identifier_limit: i64, + artifact_label: &str, +) -> PyResult { + fit_logical(names, identifier_limit, artifact_label) + .map_err(|message| shared_error(py, message)) +} diff --git a/crates/sqlbuild-python-project/src/relation_naming/main/mod.rs b/crates/sqlbuild-python-project/src/relation_naming/main/mod.rs new file mode 100644 index 0000000000..4c344f8415 --- /dev/null +++ b/crates/sqlbuild-python-project/src/relation_naming/main/mod.rs @@ -0,0 +1,3 @@ +pub mod fitted_logical_name; +pub mod register_fitting; +pub mod shared_error; diff --git a/crates/sqlbuild-python-project/src/relation_naming/main/register_fitting.rs b/crates/sqlbuild-python-project/src/relation_naming/main/register_fitting.rs new file mode 100644 index 0000000000..3660c8579c --- /dev/null +++ b/crates/sqlbuild-python-project/src/relation_naming/main/register_fitting.rs @@ -0,0 +1,13 @@ +//! Register the fitting functions and classes. + +use crate::relation_naming::_helpers::fitting::fit_artifact_logical_name; +use crate::relation_naming::_helpers::fitting::fit_auxiliary_relation_name; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the fitting functions and classes. +pub fn register_fitting(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(fit_artifact_logical_name, module)?)?; + module.add_function(wrap_pyfunction!(fit_auxiliary_relation_name, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/relation_naming/main/shared_error.rs b/crates/sqlbuild-python-project/src/relation_naming/main/shared_error.rs new file mode 100644 index 0000000000..ac8d6d4c04 --- /dev/null +++ b/crates/sqlbuild-python-project/src/relation_naming/main/shared_error.rs @@ -0,0 +1,10 @@ +//! Shared error. + +use pyo3::prelude::{PyErr, Python}; +use sqlbuild_host::entries::models::FacadeHost; +use sqlbuild_python_base::native_host::main::host_error::host_error; + +/// Shared error. +pub fn shared_error(py: Python<'_>, message: String) -> PyErr { + host_error(py, FacadeHost::SharedInputError, &message) +} diff --git a/crates/sqlbuild-python-project/src/relation_naming/mod.rs b/crates/sqlbuild-python-project/src/relation_naming/mod.rs new file mode 100644 index 0000000000..fdc0964b53 --- /dev/null +++ b/crates/sqlbuild-python-project/src/relation_naming/mod.rs @@ -0,0 +1,4 @@ +//! Fitting relation names to warehouse limits. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_reads.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_reads.rs new file mode 100644 index 0000000000..ba9a66a5db --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_reads.rs @@ -0,0 +1,19 @@ +//! A compiled audit's attributes the planner reads, from its native view. + +use pyo3::prelude::{Bound, PyAny, PyResult}; + +use crate::resource_views::models::NativeCompiledAudit; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; + +/// The native view behind an audit object; any other object is a Python-built record, counted. +#[inline] +pub(crate) fn native_audit<'a>( + audit: &'a Bound<'_, PyAny>, +) -> PyResult> { + if let Ok(view) = audit.cast::() { + return Ok(Some(view.get())); + } + count_stage(audit.py(), COMPILED_VIEW_FALLBACKS, 1)?; + Ok(None) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs similarity index 70% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs rename to crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs index 5b08d91959..5eafc3e79a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs @@ -1,20 +1,21 @@ //! Audits attached by the native loop, held by the compiled project, and their views. +use crate::resource_views::models::NativeAuditInput; +use crate::resource_views::models::NativeCompiledAudit; +use crate::resource_views::types::AttachedAudit; +use crate::resource_views::types::AuditRecord; use pyo3::exceptions::{PyIndexError, PyRuntimeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; use pyo3::types::PyTuple; -use pyo3::{FromPyObject, IntoPyObjectExt, pyclass, pymethods}; +use pyo3::{FromPyObject, pymethods}; -use sqlbuild_project::models as project_models; use sqlbuild_project::types::ObjectKeyRow; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::compiled_model_view::object_key; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - InputViewClasses, reference_tuple, view_classes, -}; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::main::compiled_models::object_key::object_key; +use crate::compiled_views::main::model_inputs::reference_tuple::reference_tuple; +use crate::compiled_views::main::model_inputs::view_classes::view_classes; +use crate::compiled_views::models::InputViewClasses; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::scopes::main::reports::json_to_python::json_to_python; @@ -37,44 +38,7 @@ pub(crate) type AuditAssemblyRow = ( Option<(String, String)>, ); -/// What the audit loop attached for one singular audit block or attached generic audit. -pub(crate) type AttachedAudit = project_models::AttachedAudit; - -/// One attached audit held by the compiled project. -pub(crate) type AuditRecord = project_models::AuditRecord; - -/// An attached audit's user values. -pub(crate) struct AuditHandles { - /// The discovered (or built-in) audit file and block the SQL comes from. - pub(crate) audit_file: Py, - pub(crate) audit_block: Py, - /// The authored measurement policy specs, carrying user values. - pub(crate) thresholds: Py, - pub(crate) minimum_samples: Py, - pub(crate) evidence_limit: Py, - /// Usage records, the identities the scope index is built from. - pub(crate) usages: Py, -} - -/// An audit record's Python projections, each built once on first access. -pub(crate) struct AuditProjections { - pub(crate) references: PyOnceLock>, - pub(crate) key: PyOnceLock>, -} - -/// The audit record of an attached audit, before assembly resolves its scope dependencies. -pub(crate) fn audit_record(attached: AttachedAudit) -> AuditRecord { - AuditRecord { - attached, - scope_deps: None, - host: AuditProjections { - references: PyOnceLock::new(), - key: PyOnceLock::new(), - }, - } -} - -fn read_audit<'py, T>( +pub(crate) fn read_audit<'py, T>( owner: &Py, index: usize, py: Python<'py>, @@ -208,7 +172,7 @@ macro_rules! audit_getters { } #[getter] - pub(crate) fn attached_target_name(&self, py: Python<'_>) -> PyResult> { + pub fn attached_target_name(&self, py: Python<'_>) -> PyResult> { read_audit(&self.owner, self.index, py, |audit, _| { Ok(audit.attached.target.as_ref().map(|target| target.name.clone())) }) @@ -267,19 +231,6 @@ macro_rules! audit_getters { }; } -/// One attached audit, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeAuditInput { - owner: Py, - index: usize, -} - -impl NativeAuditInput { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } -} - audit_getters!(NativeAuditInput, { #[getter] pub(crate) fn declaration_usages(&self, py: Python<'_>) -> PyResult> { @@ -322,19 +273,6 @@ audit_getters!(NativeAuditInput, { } }); -/// One assembled audit, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeCompiledAudit { - owner: Py, - index: usize, -} - -impl NativeCompiledAudit { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } -} - audit_getters!(NativeCompiledAudit, { /// The audit's planner-facing `CompiledObjectKey`. #[getter] @@ -386,54 +324,3 @@ audit_getters!(NativeCompiledAudit, { }) } }); - -/// A native audit view's reference rows, or `None` for a hand-built audit record. -pub(crate) fn audit_reference_rows( - audit: &Bound<'_, PyAny>, -) -> PyResult>> { - let py: Python<'_> = audit.py(); - let (owner, index): (&Py, usize) = - if let Ok(view) = audit.cast::() { - (&view.get().owner, view.get().index) - } else if let Ok(view) = audit.cast::() { - (&view.get().owner, view.get().index) - } else { - return Ok(None); - }; - read_audit(owner, index, py, |record, _| { - Ok(Some(record.attached.references.clone())) - }) -} - -impl NativeCompiledAudit { - /// The `CompiledAudit` field values, in field order, without building the record. - pub(crate) fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { - Ok(vec![ - self.key(py)?.into_bound(py), - self.scope_deps(py)?.into_any(), - self.name(py)?.into_bound_py_any(py)?, - self.definition_name(py)?.into_bound_py_any(py)?, - self.audit_file(py)?.into_bound(py), - self.audit_block(py)?.into_bound(py), - self.sql_body(py)?.into_bound_py_any(py)?, - self.evaluation_mode(py)?.into_bound(py), - self.measurement_contract(py)?.into_bound(py), - self.thresholds(py)?.into_bound(py), - self.minimum_samples(py)?.into_bound(py), - self.measure_sql(py)?.into_bound_py_any(py)?, - self.evidence_sql(py)?.into_bound_py_any(py)?, - self.evidence_limit(py)?.into_bound(py), - self.references(py)?.into_bound(py).into_any(), - self.attached_target_kind(py)?, - self.attached_target_name(py)?.into_bound_py_any(py)?, - self.attached_column_name(py)?.into_bound_py_any(py)?, - self.severity(py)?, - self.run_scope(py)?.into_bound_py_any(py)?, - pyo3::types::PyBool::new(py, self.always_run(py)?) - .to_owned() - .into_any(), - self.description(py)?.into_bound_py_any(py)?, - self.meta(py)?.into_bound(py), - ]) - } -} diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/function_reads.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/function_reads.rs new file mode 100644 index 0000000000..0584e54ae8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/function_reads.rs @@ -0,0 +1,19 @@ +//! A compiled function's attributes the planner reads, from its native view. + +use pyo3::prelude::{Bound, PyAny, PyResult}; + +use crate::resource_views::models::NativeCompiledFunction; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; + +/// The native view behind a function object; any other object is a Python-built record, counted. +#[inline] +pub(crate) fn native_function<'a>( + function: &'a Bound<'_, PyAny>, +) -> PyResult> { + if let Ok(view) = function.cast::() { + return Ok(Some(view.get())); + } + count_stage(function.py(), COMPILED_VIEW_FALLBACKS, 1)?; + Ok(None) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs similarity index 73% rename from crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs rename to crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs index bdeb2eb1cf..f6e5c2a22c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs @@ -1,24 +1,25 @@ //! Functions attached by the native loop, held by the compiled project, and their views. +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeFunctionInput; +use crate::resource_views::types::AttachedFunction; +use crate::resource_views::types::FunctionRecord; use pyo3::exceptions::{PyIndexError, PyRuntimeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{FromPyObject, IntoPyObjectExt, pyclass, pymethods}; +use pyo3::{FromPyObject, pymethods}; use sqlbuild_python_base::positional_record; -use sqlbuild_project::models as project_models; use sqlbuild_project::types::{NamedTypeRow, ObjectKeyRow}; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - defaulted_destination, object_key, relation_location, -}; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - InputViewClasses, reference_tuple, view_classes, -}; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::graph::project_graph::ResourceRow; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::main::compiled_models::defaulted_destination::defaulted_destination; +use crate::compiled_views::main::compiled_models::object_key::object_key; +use crate::compiled_views::main::compiled_models::relation_location::relation_location; +use crate::compiled_views::main::model_inputs::reference_tuple::reference_tuple; +use crate::compiled_views::main::model_inputs::view_classes::view_classes; +use crate::compiled_views::models::InputViewClasses; use sqlbuild_project::models::DestinationFact; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -27,7 +28,7 @@ const SQL_LANGUAGE: &str = "sql"; const PYTHON_LANGUAGE: &str = "python"; const FUNCTION_FILE_ATTRIBUTE: &str = "relative_path"; const FILE_PATH_ATTRIBUTE: &str = "file_path"; -const NOT_ASSEMBLED: &str = "the function was not assembled"; +pub(crate) const NOT_ASSEMBLED: &str = "the function was not assembled"; const FUNCTION_CLASSES: &str = "functions"; /// The Python classes a function view projects its argument and return column records with. @@ -50,42 +51,6 @@ positional_record! { } } -/// What the function loop attached for one SQL or Python function. -pub(crate) type AttachedFunction = project_models::AttachedFunction; - -/// One attached function held by the compiled project. -pub(crate) type FunctionRecord = project_models::FunctionRecord; - -/// An attached function's user values. -pub(crate) struct FunctionHandles { - /// The discovered function file. - pub(crate) function_file: Py, - /// Usage records, the identities the scope index is built from. - pub(crate) usages: Py, -} - -/// A function record's Python projections, each built once on first access. -pub(crate) struct FunctionProjections { - arguments: PyOnceLock>, - return_columns: PyOnceLock>, - pub(crate) references: PyOnceLock>, - pub(crate) key: PyOnceLock>, -} - -/// The function record of an attached function, before assembly resolves its dependencies. -pub(crate) fn function_record(attached: AttachedFunction) -> FunctionRecord { - FunctionRecord { - attached, - deps: None, - host: FunctionProjections { - arguments: PyOnceLock::new(), - return_columns: PyOnceLock::new(), - references: PyOnceLock::new(), - key: PyOnceLock::new(), - }, - } -} - /// `rows` as `class` records, built once into `cache`. fn named_types( rows: &[NamedTypeRow], @@ -108,7 +73,7 @@ fn named_types( } /// The function's `FunctionLanguage` member. -fn language<'py>( +pub(crate) fn language<'py>( function: &FunctionRecord, classes: &FunctionClasses<'py>, ) -> PyResult> { @@ -120,7 +85,7 @@ fn language<'py>( classes.function_language.call1((value,)) } -fn read_function<'py, T>( +pub(crate) fn read_function<'py, T>( owner: &Py, index: usize, py: Python<'py>, @@ -269,19 +234,6 @@ macro_rules! function_getters { }; } -/// One attached function, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeFunctionInput { - owner: Py, - index: usize, -} - -impl NativeFunctionInput { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } -} - function_getters!(NativeFunctionInput, { #[getter] pub(crate) fn function_file(&self, py: Python<'_>) -> PyResult> { @@ -387,72 +339,6 @@ function_getters!(NativeFunctionInput, { } }); -/// One assembled function, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeCompiledFunction { - owner: Py, - index: usize, - /// The destination and fingerprint destination with target defaults applied. - defaulted: Option<(DestinationFact, DestinationFact)>, - destination: PyOnceLock>, - fingerprint_destination: PyOnceLock>, -} - -impl NativeCompiledFunction { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self::defaulted(owner, index, None) - } - - /// The function's project graph row: its key, the resources it reads and its tags. - pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { - read_function(&self.owner, self.index, py, |function, _| { - let deps: &[ObjectKeyRow] = function - .deps - .as_deref() - .ok_or_else(|| PyRuntimeError::new_err(NOT_ASSEMBLED))?; - Ok(ResourceRow { - key: ( - function.attached.node_type().to_owned(), - function.attached.name.clone(), - ), - deps: deps.to_vec(), - tags: function.attached.tags.clone(), - folder: None, - }) - }) - } - - fn defaulted( - owner: Py, - index: usize, - defaulted: Option<(DestinationFact, DestinationFact)>, - ) -> Self { - Self { - owner, - index, - defaulted, - destination: PyOnceLock::new(), - fingerprint_destination: PyOnceLock::new(), - } - } - - /// The destination and fingerprint destination, with target defaults once applied. - pub(crate) fn current_destinations( - &self, - py: Python<'_>, - ) -> PyResult<(DestinationFact, DestinationFact)> { - if let Some(defaulted) = &self.defaulted { - return Ok(defaulted.clone()); - } - read_function(&self.owner, self.index, py, |function, _| { - Ok(( - function.attached.destination.clone(), - function.attached.fingerprint_destination.clone(), - )) - }) - } -} - function_getters!(NativeCompiledFunction, { /// The function's planner-facing `CompiledObjectKey`. #[getter] @@ -535,7 +421,7 @@ function_getters!(NativeCompiledFunction, { } /// This function with target defaults; a non-inheriting Python function is name-qualified. - pub(crate) fn with_target_defaults( + pub fn with_target_defaults( &self, py: Python<'_>, defaults: (Option, Option), @@ -562,29 +448,3 @@ function_getters!(NativeCompiledFunction, { )) } }); - -impl NativeCompiledFunction { - /// The `CompiledFunction` field values, in field order, without building the record. - pub(crate) fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { - Ok(vec![ - self.key(py)?.into_bound(py), - self.deps(py)?.into_any(), - self.name(py)?.into_bound_py_any(py)?, - self.relative_path(py)?.into_bound(py), - self.arguments(py)?.into_bound(py).into_any(), - self.returns(py)?.into_bound_py_any(py)?, - self.body_sql(py)?.into_bound_py_any(py)?, - self.destination(py)?.into_bound(py), - self.fingerprint_destination(py)?.into_bound(py), - self.return_columns(py)?.into_bound(py).into_any(), - self.references(py)?.into_bound(py).into_any(), - self.language(py)?, - self.source_file_path(py)?.into_bound(py), - self.runtime_version(py)?.into_bound_py_any(py)?, - self.entry_point(py)?.into_bound_py_any(py)?, - self.packages(py)?.into_any(), - self.tags(py)?.into_any(), - self.description(py)?.into_bound_py_any(py)?, - ]) - } -} diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/mod.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/mod.rs new file mode 100644 index 0000000000..fe40ad34dc --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod audit_reads; +pub(crate) mod audit_views; +pub(crate) mod function_reads; +pub(crate) mod function_views; +pub(crate) mod seed_reads; +pub(crate) mod seed_views; +pub(crate) mod source_reads; +pub(crate) mod source_views; diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_reads.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_reads.rs new file mode 100644 index 0000000000..7bce632019 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_reads.rs @@ -0,0 +1,19 @@ +//! A compiled seed's attributes the planner reads, from its native view. + +use pyo3::prelude::{Bound, PyAny, PyResult}; + +use crate::resource_views::models::NativeCompiledSeed; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; + +/// The native view behind a seed object; any other object is a Python-built record, counted. +#[inline] +pub(crate) fn native_seed<'a>( + seed: &'a Bound<'_, PyAny>, +) -> PyResult> { + if let Ok(view) = seed.cast::() { + return Ok(Some(view.get())); + } + count_stage(seed.py(), COMPILED_VIEW_FALLBACKS, 1)?; + Ok(None) +} diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs new file mode 100644 index 0000000000..cf99e00a71 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs @@ -0,0 +1,60 @@ +//! Seeds paired with their declarations, held by the compiled project, and their views. + +use crate::resource_views::models::SeedNamespace; +use crate::resource_views::models::SeedParts; +use crate::resource_views::types::SeedRecord; +use crate::resource_views::types::SeedViews; +use pyo3::exceptions::PyIndexError; +use pyo3::prelude::{Py, PyResult, Python}; + +use crate::compiled_project::models::NativeCompiledProject; +use sqlbuild_project::models::DestinationFact; + +pub(crate) const SEED_RESOURCE: &str = "seed"; +pub(crate) const TAGS_ATTRIBUTE: &str = "tags"; +pub(crate) const NAME_ATTRIBUTE: &str = "name"; +pub(crate) const FILE_PATH_ATTRIBUTE: &str = "file_path"; +pub(crate) const STEM_ATTRIBUTE: &str = "stem"; +pub(crate) const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; + +impl SeedViews for SeedRecord { + fn parts(&self) -> SeedParts<'_> { + SeedParts { + name: &self.name, + seed_file: &self.host.seed_file, + schema_entry: &self.host.schema_entry, + schema_file: &self.host.schema_file, + } + } + + fn set_namespace(&mut self, namespace: SeedNamespace) { + let SeedNamespace { + database, + schema, + logical_database, + logical_schema, + } = namespace; + self.destination = Some(DestinationFact { + database, + schema, + name: self.name.clone(), + qualified_name: None, + logical_schema, + logical_database, + }); + } +} + +pub(crate) fn read_seed<'py, T>( + owner: &Py, + index: usize, + py: Python<'py>, + read: impl FnOnce(&SeedRecord, Python<'py>) -> PyResult, +) -> PyResult { + let owner = owner.bind(py).borrow(); + let seed: &SeedRecord = owner + .seeds() + .get(index) + .ok_or_else(|| PyIndexError::new_err(index))?; + read(seed, py) +} diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/source_reads.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/source_reads.rs new file mode 100644 index 0000000000..ee72b3e2a5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/source_reads.rs @@ -0,0 +1,19 @@ +//! A compiled source's attributes the planner reads, from its native view. + +use pyo3::prelude::{Bound, PyAny, PyResult}; + +use crate::resource_views::models::NativeCompiledSource; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; + +/// The native view behind a source object; any other object is a Python-built record, counted. +#[inline] +pub(crate) fn native_source<'a>( + source: &'a Bound<'_, PyAny>, +) -> PyResult> { + if let Ok(view) = source.cast::() { + return Ok(Some(view.get())); + } + count_stage(source.py(), COMPILED_VIEW_FALLBACKS, 1)?; + Ok(None) +} diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/source_views.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/source_views.rs new file mode 100644 index 0000000000..b584ce0ed0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/source_views.rs @@ -0,0 +1,124 @@ +//! Sources attached by the native loop, held by the compiled project, and their views. + +use crate::resource_views::main::source_views::replaced::replaced; +use crate::resource_views::models::NativeCompiledSource; +use crate::resource_views::models::NativeSourceInput; +use crate::resource_views::models::SourceKeyFacts; +use crate::resource_views::models::SourceProjections; +use crate::resource_views::types::AttachedSource; +use crate::resource_views::types::SourceRecord; +use crate::resource_views::types::SourceViews; +use pyo3::exceptions::PyIndexError; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::positional_record; + +use crate::compiled_project::main::python_column_facts::python_column_facts; +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::main::compiled_models::audit_key_rows::audit_key_rows; + +pub(crate) const SOURCE_RESOURCE: &str = "source"; + +positional_record! { + into; + /// `(name, has a loader, database, schema)` of one source. + pub(crate) struct SourceAssemblyRow { + pub(crate) name: String, + pub(crate) managed: bool, + pub(crate) database: Option, + pub(crate) schema: Option, + } +} + +/// The source record of an attached source, before assembly resolves its namespace. +pub(crate) fn source_record(attached: AttachedSource) -> SourceRecord { + SourceRecord { + attached, + namespace: None, + host: SourceProjections { + key_facts: PyOnceLock::new(), + attached_entry: PyOnceLock::new(), + assembled_entry: PyOnceLock::new(), + key: PyOnceLock::new(), + }, + } +} + +impl SourceViews for SourceRecord { + fn key_facts(&self, py: Python<'_>) -> PyResult<&SourceKeyFacts> { + self.host.key_facts.get_or_try_init(py, || { + Ok(SourceKeyFacts { + columns: python_column_facts(self.attached.host.columns.bind(py).as_any(), None)?, + audits: audit_key_rows(self.attached.host.audits.bind(py).as_any())?, + }) + }) + } + + fn attached_entry(&self, py: Python<'_>) -> PyResult> { + self.host + .attached_entry + .get_or_try_init(py, || { + let source: &AttachedSource = &self.attached; + let changes: Bound<'_, PyDict> = PyDict::new(py); + changes.set_item("contract", &source.contract)?; + changes.set_item("database", &source.database)?; + changes.set_item("schema", &source.schema)?; + changes.set_item("table", &source.table)?; + changes.set_item("expression", &source.expression)?; + changes.set_item("description", &source.description)?; + changes.set_item("meta", source.host.meta.bind(py))?; + changes.set_item("columns", source.host.columns.bind(py))?; + changes.set_item("audits", source.host.audits.bind(py))?; + replaced(py, source.host.entry.bind(py), &changes) + }) + .map(|entry| entry.clone_ref(py)) + } + + fn assembled_entry(&self, py: Python<'_>) -> PyResult> { + let attached: Py = self.attached_entry(py)?; + let Some(Some((database, schema))) = &self.namespace else { + return Ok(attached); + }; + self.host + .assembled_entry + .get_or_try_init(py, || { + let changes: Bound<'_, PyDict> = PyDict::new(py); + changes.set_item("database", database)?; + changes.set_item("schema", schema)?; + replaced(py, attached.bind(py), &changes) + }) + .map(|entry| entry.clone_ref(py)) + } +} + +/// Read the source record behind a native source view, or `None` for any other object. +pub(crate) fn read_source_view<'py, T>( + view: &Bound<'py, PyAny>, + read: impl FnOnce(&SourceRecord, Python<'py>) -> PyResult, +) -> Option> { + let py: Python<'py> = view.py(); + if let Ok(input) = view.cast::() { + let input: &NativeSourceInput = input.get(); + return Some(read_source(&input.owner, input.index, py, read)); + } + let Ok(compiled) = view.cast::() else { + return None; + }; + let compiled: &NativeCompiledSource = compiled.get(); + Some(read_source(&compiled.owner, compiled.index, py, read)) +} + +pub(crate) fn read_source<'py, T>( + owner: &Py, + index: usize, + py: Python<'py>, + read: impl FnOnce(&SourceRecord, Python<'py>) -> PyResult, +) -> PyResult { + let owner = owner.bind(py).borrow(); + let source: &SourceRecord = owner + .sources() + .get(index) + .ok_or_else(|| PyIndexError::new_err(index))?; + read(source, py) +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/always_run.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/always_run.rs new file mode 100644 index 0000000000..f5c1f81f6b --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/always_run.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `always_run`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `always_run`, as its Python attribute reads it. +pub fn always_run<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .always_run(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "always_run") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_column_name.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_column_name.rs new file mode 100644 index 0000000000..fbc352a063 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_column_name.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `attached_column_name`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `attached_column_name`, as its Python attribute reads it. +pub fn attached_column_name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .attached_column_name(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "attached_column_name") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_kind.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_kind.rs new file mode 100644 index 0000000000..368a605af4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_kind.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `attached_target_kind`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `attached_target_kind`, as its Python attribute reads it. +pub fn attached_target_kind<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .attached_target_kind(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "attached_target_kind") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_name.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_name.rs new file mode 100644 index 0000000000..174817d0bf --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_name.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `attached_target_name`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `attached_target_name`, as its Python attribute reads it. +pub fn attached_target_name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .attached_target_name(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "attached_target_name") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/definition_name.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/definition_name.rs new file mode 100644 index 0000000000..ba5aa9b32e --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/definition_name.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `definition_name`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `definition_name`, as its Python attribute reads it. +pub fn definition_name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .definition_name(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "definition_name") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/description.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/description.rs new file mode 100644 index 0000000000..c2015c53f8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/description.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `description`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `description`, as its Python attribute reads it. +pub fn description<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .description(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "description") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evaluation_mode.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evaluation_mode.rs new file mode 100644 index 0000000000..9a0049b3ee --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evaluation_mode.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `evaluation_mode`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `evaluation_mode`, as its Python attribute reads it. +pub fn evaluation_mode<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .evaluation_mode(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "evaluation_mode") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_limit.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_limit.rs new file mode 100644 index 0000000000..84f5115c47 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_limit.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `evidence_limit`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `evidence_limit`, as its Python attribute reads it. +pub fn evidence_limit<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .evidence_limit(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "evidence_limit") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_sql.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_sql.rs new file mode 100644 index 0000000000..4f4fcd981c --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_sql.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `evidence_sql`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `evidence_sql`, as its Python attribute reads it. +pub fn evidence_sql<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .evidence_sql(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "evidence_sql") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/key.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/key.rs new file mode 100644 index 0000000000..4fc7b3f646 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/key.rs @@ -0,0 +1,15 @@ +//! The compiled audit's `key`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `key`, as its Python attribute reads it. +pub fn key<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "key") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/measurement_contract.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/measurement_contract.rs new file mode 100644 index 0000000000..c78b6edb17 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/measurement_contract.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `measurement_contract`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `measurement_contract`, as its Python attribute reads it. +pub fn measurement_contract<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .measurement_contract(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "measurement_contract") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/meta.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/meta.rs new file mode 100644 index 0000000000..5338add72a --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/meta.rs @@ -0,0 +1,15 @@ +//! The compiled audit's `meta`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `meta`, as its Python attribute reads it. +pub fn meta<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view.meta(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "meta") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/minimum_samples.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/minimum_samples.rs new file mode 100644 index 0000000000..3c6543f8ab --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/minimum_samples.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `minimum_samples`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `minimum_samples`, as its Python attribute reads it. +pub fn minimum_samples<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .minimum_samples(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "minimum_samples") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/mod.rs new file mode 100644 index 0000000000..16994fd38a --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/mod.rs @@ -0,0 +1,20 @@ +pub mod always_run; +pub mod attached_column_name; +pub mod attached_target_kind; +pub mod attached_target_name; +pub mod definition_name; +pub mod description; +pub mod evaluation_mode; +pub mod evidence_limit; +pub mod evidence_sql; +pub mod key; +pub mod measurement_contract; +pub mod meta; +pub mod minimum_samples; +pub mod name; +pub mod references; +pub mod run_scope; +pub mod scope_deps; +pub mod severity; +pub mod sql_body; +pub mod thresholds; diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/name.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/name.rs new file mode 100644 index 0000000000..1b0f2e7c9b --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/name.rs @@ -0,0 +1,15 @@ +//! The compiled audit's `name`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `name`, as its Python attribute reads it. +pub fn name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "name") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/references.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/references.rs new file mode 100644 index 0000000000..d5e72183f9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/references.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `references`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `references`, as its Python attribute reads it. +pub fn references<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .references(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "references") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/run_scope.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/run_scope.rs new file mode 100644 index 0000000000..625a5dc1be --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/run_scope.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `run_scope`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `run_scope`, as its Python attribute reads it. +pub fn run_scope<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .run_scope(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "run_scope") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/scope_deps.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/scope_deps.rs new file mode 100644 index 0000000000..942d44339e --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/scope_deps.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `scope_deps`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `scope_deps`, as its Python attribute reads it. +pub fn scope_deps<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .scope_deps(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "scope_deps") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/severity.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/severity.rs new file mode 100644 index 0000000000..13d47a3965 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/severity.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `severity`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `severity`, as its Python attribute reads it. +pub fn severity<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .severity(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "severity") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/sql_body.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/sql_body.rs new file mode 100644 index 0000000000..9d7c183eaf --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/sql_body.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `sql_body`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `sql_body`, as its Python attribute reads it. +pub fn sql_body<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .sql_body(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "sql_body") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/thresholds.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/thresholds.rs new file mode 100644 index 0000000000..3082b33e51 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_reads/thresholds.rs @@ -0,0 +1,17 @@ +//! The compiled audit's `thresholds`, as its Python attribute reads it. + +use crate::resource_views::_helpers::audit_reads::native_audit; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled audit's `thresholds`, as its Python attribute reads it. +pub fn thresholds<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + if let Some(view) = native_audit(audit)? { + return view + .thresholds(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(audit, "thresholds") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_record.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_record.rs new file mode 100644 index 0000000000..33cefb79c0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_record.rs @@ -0,0 +1,18 @@ +//! The audit record of an attached audit, before assembly resolves its scope dependencies. + +use crate::resource_views::models::AuditProjections; +use crate::resource_views::types::AttachedAudit; +use crate::resource_views::types::AuditRecord; +use pyo3::sync::PyOnceLock; + +/// The audit record of an attached audit, before assembly resolves its scope dependencies. +pub fn audit_record(attached: AttachedAudit) -> AuditRecord { + AuditRecord { + attached, + scope_deps: None, + host: AuditProjections { + references: PyOnceLock::new(), + key: PyOnceLock::new(), + }, + } +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_reference_rows.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_reference_rows.rs new file mode 100644 index 0000000000..c3efd47c89 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_views/audit_reference_rows.rs @@ -0,0 +1,24 @@ +//! A native audit view's reference rows, or `None` for a hand-built audit record. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::resource_views::_helpers::audit_views::read_audit; +use crate::resource_views::models::NativeAuditInput; +use crate::resource_views::models::NativeCompiledAudit; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use sqlbuild_project::models::ReferenceRow; + +/// A native audit view's reference rows, or `None` for a hand-built audit record. +pub fn audit_reference_rows(audit: &Bound<'_, PyAny>) -> PyResult>> { + let py: Python<'_> = audit.py(); + let (owner, index): (&Py, usize) = + if let Ok(view) = audit.cast::() { + (&view.get().owner, view.get().index) + } else if let Ok(view) = audit.cast::() { + (&view.get().owner, view.get().index) + } else { + return Ok(None); + }; + read_audit(owner, index, py, |record, _| { + Ok(Some(record.attached.references.clone())) + }) +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/audit_views/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/audit_views/mod.rs new file mode 100644 index 0000000000..598f39715c --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/audit_views/mod.rs @@ -0,0 +1,2 @@ +pub mod audit_record; +pub mod audit_reference_rows; diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/arguments.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/arguments.rs new file mode 100644 index 0000000000..5fb05b4c33 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/arguments.rs @@ -0,0 +1,17 @@ +//! The compiled function's `arguments`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `arguments`, as its Python attribute reads it. +pub fn arguments<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .arguments(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "arguments") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/body_sql.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/body_sql.rs new file mode 100644 index 0000000000..19cbb14b89 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/body_sql.rs @@ -0,0 +1,17 @@ +//! The compiled function's `body_sql`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `body_sql`, as its Python attribute reads it. +pub fn body_sql<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .body_sql(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "body_sql") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/deps.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/deps.rs new file mode 100644 index 0000000000..cf40148ff3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/deps.rs @@ -0,0 +1,15 @@ +//! The compiled function's `deps`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `deps`, as its Python attribute reads it. +pub fn deps<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view.deps(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "deps") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/destination.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/destination.rs new file mode 100644 index 0000000000..414d0c0217 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/destination.rs @@ -0,0 +1,17 @@ +//! The compiled function's `destination`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `destination`, as its Python attribute reads it. +pub fn destination<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .destination(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "destination") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/entry_point.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/entry_point.rs new file mode 100644 index 0000000000..4af4e43095 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/entry_point.rs @@ -0,0 +1,17 @@ +//! The compiled function's `entry_point`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `entry_point`, as its Python attribute reads it. +pub fn entry_point<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .entry_point(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "entry_point") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/fingerprint_destination.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/fingerprint_destination.rs new file mode 100644 index 0000000000..5841da1258 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/fingerprint_destination.rs @@ -0,0 +1,17 @@ +//! The compiled function's `fingerprint_destination`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `fingerprint_destination`, as its Python attribute reads it. +pub fn fingerprint_destination<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .fingerprint_destination(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "fingerprint_destination") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/key.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/key.rs new file mode 100644 index 0000000000..0b37891c16 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/key.rs @@ -0,0 +1,15 @@ +//! The compiled function's `key`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `key`, as its Python attribute reads it. +pub fn key<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "key") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/language.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/language.rs new file mode 100644 index 0000000000..76218bd9fc --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/language.rs @@ -0,0 +1,17 @@ +//! The compiled function's `language`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `language`, as its Python attribute reads it. +pub fn language<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .language(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "language") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/mod.rs new file mode 100644 index 0000000000..04cf303460 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/mod.rs @@ -0,0 +1,15 @@ +pub mod arguments; +pub mod body_sql; +pub mod deps; +pub mod destination; +pub mod entry_point; +pub mod fingerprint_destination; +pub mod key; +pub mod language; +pub mod name; +pub mod packages; +pub mod relative_path; +pub mod return_columns; +pub mod returns; +pub mod runtime_version; +pub mod source_file_path; diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/name.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/name.rs new file mode 100644 index 0000000000..a58608ab5e --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/name.rs @@ -0,0 +1,15 @@ +//! The compiled function's `name`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `name`, as its Python attribute reads it. +pub fn name<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "name") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/packages.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/packages.rs new file mode 100644 index 0000000000..b019e85eff --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/packages.rs @@ -0,0 +1,17 @@ +//! The compiled function's `packages`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `packages`, as its Python attribute reads it. +pub fn packages<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .packages(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "packages") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/relative_path.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/relative_path.rs new file mode 100644 index 0000000000..12ceaec713 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/relative_path.rs @@ -0,0 +1,17 @@ +//! The compiled function's `relative_path`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `relative_path`, as its Python attribute reads it. +pub fn relative_path<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .relative_path(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "relative_path") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/return_columns.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/return_columns.rs new file mode 100644 index 0000000000..5e40a8f43f --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/return_columns.rs @@ -0,0 +1,17 @@ +//! The compiled function's `return_columns`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `return_columns`, as its Python attribute reads it. +pub fn return_columns<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .return_columns(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "return_columns") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/returns.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/returns.rs new file mode 100644 index 0000000000..f3ce015997 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/returns.rs @@ -0,0 +1,17 @@ +//! The compiled function's `returns`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `returns`, as its Python attribute reads it. +pub fn returns<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .returns(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "returns") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/runtime_version.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/runtime_version.rs new file mode 100644 index 0000000000..6206a31407 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/runtime_version.rs @@ -0,0 +1,17 @@ +//! The compiled function's `runtime_version`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `runtime_version`, as its Python attribute reads it. +pub fn runtime_version<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .runtime_version(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "runtime_version") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/source_file_path.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/source_file_path.rs new file mode 100644 index 0000000000..a5bcd85d13 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/source_file_path.rs @@ -0,0 +1,17 @@ +//! The compiled function's `source_file_path`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `source_file_path`, as its Python attribute reads it. +pub fn source_file_path<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .source_file_path(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "source_file_path") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_views/function_record.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_views/function_record.rs new file mode 100644 index 0000000000..1090fa0106 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_views/function_record.rs @@ -0,0 +1,20 @@ +//! The function record of an attached function, before assembly resolves its dependencies. + +use crate::resource_views::models::FunctionProjections; +use crate::resource_views::types::AttachedFunction; +use crate::resource_views::types::FunctionRecord; +use pyo3::sync::PyOnceLock; + +/// The function record of an attached function, before assembly resolves its dependencies. +pub fn function_record(attached: AttachedFunction) -> FunctionRecord { + FunctionRecord { + attached, + deps: None, + host: FunctionProjections { + arguments: PyOnceLock::new(), + return_columns: PyOnceLock::new(), + references: PyOnceLock::new(), + key: PyOnceLock::new(), + }, + } +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_views/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_views/mod.rs new file mode 100644 index 0000000000..6d63b8eef2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_views/mod.rs @@ -0,0 +1 @@ +pub mod function_record; diff --git a/crates/sqlbuild-python-project/src/resource_views/main/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/mod.rs new file mode 100644 index 0000000000..97fbabd537 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/mod.rs @@ -0,0 +1,9 @@ +pub mod audit_reads; +pub mod audit_views; +pub mod function_reads; +pub mod function_views; +pub mod resource_reads; +pub mod seed_reads; +pub mod seed_views; +pub mod source_reads; +pub mod source_views; diff --git a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/destination.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/destination.rs new file mode 100644 index 0000000000..1ea70b4ed1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/destination.rs @@ -0,0 +1,35 @@ +//! A resource's `destination` from whichever native view it is; any other object is read by attribute, counted. + +use crate::compiled_views::models::NativeCompiledModel; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A resource's `destination` from whichever native view it is; any other object is read by attribute, counted. +pub fn destination<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = resource.py(); + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledModel = view.get(); + return view + .destination(py) + .and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledFunction = view.get(); + return view + .destination(py) + .and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledSeed = view.get(); + return view + .destination(py) + .and_then(|value| value.into_bound_py_any(py)); + } + count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; + native_field(resource, "destination") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/graph_row.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/graph_row.rs new file mode 100644 index 0000000000..a97c37e812 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/graph_row.rs @@ -0,0 +1,40 @@ +//! A resource's graph facts without constructing its public keys or dependency tuple. + +use crate::compiled_views::main::compiled_models::model_graph_row::model_graph_row; +use crate::compiled_views::models::NativeCompiledModel; +use crate::graph::main::nodes::native_key::native_key; +use crate::graph::models::ResourceRow; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A resource's graph facts without constructing its public keys or dependency tuple. +pub fn graph_row(resource: &Bound<'_, PyAny>) -> PyResult { + let py = resource.py(); + if let Ok(view) = resource.cast::() { + return model_graph_row(view.get(), py); + } + if let Ok(view) = resource.cast::() { + return view.get().graph_row(py); + } + if let Ok(view) = resource.cast::() { + return view.get().graph_row(py); + } + if let Ok(view) = resource.cast::() { + return view.get().graph_row(py); + } + count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; + Ok(ResourceRow { + key: native_key(&native_field(resource, "key")?)?, + deps: native_field(resource, "deps")? + .try_iter()? + .map(|key| native_key(&key?)) + .collect::>()?, + tags: Vec::new(), + folder: None, + }) +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/key.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/key.rs new file mode 100644 index 0000000000..00c533c132 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/key.rs @@ -0,0 +1,34 @@ +//! A resource's `key` from whichever native view it is; any other object is read by attribute, counted. + +use crate::compiled_views::models::NativeCompiledModel; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A resource's `key` from whichever native view it is; any other object is read by attribute, counted. +pub fn key<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = resource.py(); + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledModel = view.get(); + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledFunction = view.get(); + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledSeed = view.get(); + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledSource = view.get(); + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; + native_field(resource, "key") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/mod.rs new file mode 100644 index 0000000000..0fc77e0a6d --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/mod.rs @@ -0,0 +1,6 @@ +pub mod destination; +pub mod graph_row; +pub mod key; +pub mod name; +pub mod references; +pub mod schema_entry; diff --git a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/name.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/name.rs new file mode 100644 index 0000000000..58377f87a9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/name.rs @@ -0,0 +1,34 @@ +//! A resource's `name` from whichever native view it is; any other object is read by attribute, counted. + +use crate::compiled_views::models::NativeCompiledModel; +use crate::resource_views::models::NativeCompiledFunction; +use crate::resource_views::models::NativeCompiledSeed; +use crate::resource_views::models::NativeCompiledSource; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A resource's `name` from whichever native view it is; any other object is read by attribute, counted. +pub fn name<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = resource.py(); + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledModel = view.get(); + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledFunction = view.get(); + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledSeed = view.get(); + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledSource = view.get(); + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; + native_field(resource, "name") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/references.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/references.rs new file mode 100644 index 0000000000..b1e37c87ca --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/references.rs @@ -0,0 +1,28 @@ +//! A resource's `references` from whichever native view it is; any other object is read by attribute, counted. + +use crate::compiled_views::models::NativeCompiledModel; +use crate::resource_views::models::NativeCompiledFunction; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A resource's `references` from whichever native view it is; any other object is read by attribute, counted. +pub fn references<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = resource.py(); + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledModel = view.get(); + return view + .references(py) + .and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledFunction = view.get(); + return view + .references(py) + .and_then(|value| value.into_bound_py_any(py)); + } + count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; + native_field(resource, "references") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/schema_entry.rs b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/schema_entry.rs new file mode 100644 index 0000000000..3f99966e56 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/resource_reads/schema_entry.rs @@ -0,0 +1,28 @@ +//! A resource's `schema_entry` from whichever native view it is; any other object is read by attribute, counted. + +use crate::compiled_views::models::NativeCompiledModel; +use crate::resource_views::models::NativeCompiledSeed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; +use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// A resource's `schema_entry` from whichever native view it is; any other object is read by attribute, counted. +pub fn schema_entry<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = resource.py(); + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledModel = view.get(); + return view + .schema_entry(py) + .and_then(|value| value.into_bound_py_any(py)); + } + if let Ok(view) = resource.cast::() { + let view: &NativeCompiledSeed = view.get(); + return view + .schema_entry(py) + .and_then(|value| value.into_bound_py_any(py)); + } + count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; + native_field(resource, "schema_entry") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/destination.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/destination.rs new file mode 100644 index 0000000000..c4e83d5011 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/destination.rs @@ -0,0 +1,17 @@ +//! The compiled seed's `destination`, as its Python attribute reads it. + +use crate::resource_views::_helpers::seed_reads::native_seed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled seed's `destination`, as its Python attribute reads it. +pub fn destination<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = seed.py(); + if let Some(view) = native_seed(seed)? { + return view + .destination(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(seed, "destination") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/external.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/external.rs new file mode 100644 index 0000000000..64413b77de --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/external.rs @@ -0,0 +1,15 @@ +//! The compiled seed's `external`, as its Python attribute reads it. + +use crate::resource_views::_helpers::seed_reads::native_seed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled seed's `external`, as its Python attribute reads it. +pub fn external<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = seed.py(); + if let Some(view) = native_seed(seed)? { + return view.is_external().into_bound_py_any(py); + } + native_field(seed, "external") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/key.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/key.rs new file mode 100644 index 0000000000..2183d8bdb0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/key.rs @@ -0,0 +1,15 @@ +//! The compiled seed's `key`, as its Python attribute reads it. + +use crate::resource_views::_helpers::seed_reads::native_seed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled seed's `key`, as its Python attribute reads it. +pub fn key<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = seed.py(); + if let Some(view) = native_seed(seed)? { + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(seed, "key") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/mod.rs new file mode 100644 index 0000000000..f935135d3b --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/mod.rs @@ -0,0 +1,6 @@ +pub mod destination; +pub mod external; +pub mod key; +pub mod name; +pub mod schema_entry; +pub mod seed_file; diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/name.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/name.rs new file mode 100644 index 0000000000..7aad821a4d --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/name.rs @@ -0,0 +1,15 @@ +//! The compiled seed's `name`, as its Python attribute reads it. + +use crate::resource_views::_helpers::seed_reads::native_seed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled seed's `name`, as its Python attribute reads it. +pub fn name<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = seed.py(); + if let Some(view) = native_seed(seed)? { + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(seed, "name") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/schema_entry.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/schema_entry.rs new file mode 100644 index 0000000000..147af1cadf --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/schema_entry.rs @@ -0,0 +1,17 @@ +//! The compiled seed's `schema_entry`, as its Python attribute reads it. + +use crate::resource_views::_helpers::seed_reads::native_seed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled seed's `schema_entry`, as its Python attribute reads it. +pub fn schema_entry<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = seed.py(); + if let Some(view) = native_seed(seed)? { + return view + .schema_entry(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(seed, "schema_entry") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/seed_file.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/seed_file.rs new file mode 100644 index 0000000000..d0ad10b9d9 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_reads/seed_file.rs @@ -0,0 +1,17 @@ +//! The compiled seed's `seed_file`, as its Python attribute reads it. + +use crate::resource_views::_helpers::seed_reads::native_seed; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled seed's `seed_file`, as its Python attribute reads it. +pub fn seed_file<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = seed.py(); + if let Some(view) = native_seed(seed)? { + return view + .seed_file(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(seed, "seed_file") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_views/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_views/mod.rs new file mode 100644 index 0000000000..4ad797daa4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_views/mod.rs @@ -0,0 +1 @@ +pub mod paired_seeds; diff --git a/crates/sqlbuild-python-project/src/resource_views/main/seed_views/paired_seeds.rs b/crates/sqlbuild-python-project/src/resource_views/main/seed_views/paired_seeds.rs new file mode 100644 index 0000000000..18e88f7247 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/seed_views/paired_seeds.rs @@ -0,0 +1,66 @@ +//! Pair each declaration with the CSV file of its name, raising for the first without one. + +use crate::resource_views::_helpers::seed_views::FILE_PATH_ATTRIBUTE; +use crate::resource_views::_helpers::seed_views::NAME_ATTRIBUTE; +use crate::resource_views::_helpers::seed_views::RELATIVE_PATH_ATTRIBUTE; +use crate::resource_views::_helpers::seed_views::STEM_ATTRIBUTE; +use crate::resource_views::models::SeedHandles; +use crate::resource_views::types::SeedRecord; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use sqlbuild_attachments::seeds::main::pair_seed_declarations::pair_seed_declarations; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Pair each declaration with the CSV file of its name, raising for the first without one. +pub fn paired_seeds( + declarations: Vec<(Py, Py)>, + seed_files: &[Py], + compile_error: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = compile_error.py(); + let mut names: Vec = Vec::with_capacity(declarations.len()); + for (entry, _) in &declarations { + names.push(native_field(entry.bind(py), NAME_ATTRIBUTE)?.extract()?); + } + let mut stems: Vec = Vec::with_capacity(seed_files.len()); + for seed_file in seed_files { + stems.push( + native_field( + &native_field(seed_file.bind(py), FILE_PATH_ATTRIBUTE)?, + STEM_ATTRIBUTE, + )? + .extract()?, + ); + } + let pairs: Vec = match pair_seed_declarations(&names, &stems) { + Ok(pairs) => pairs, + Err(missing) => { + let schema_file: &Py = &declarations[missing].1; + let relative_path: String = + native_field(schema_file.bind(py), RELATIVE_PATH_ATTRIBUTE)? + .str()? + .extract()?; + let message: String = format!( + "Seed declaration '{}' in {relative_path} has no matching CSV file under seeds/", + names[missing] + ); + return Err(PyErr::from_value(compile_error.call1((message,))?)); + } + }; + let mut seeds: Vec = Vec::with_capacity(pairs.len()); + for (((schema_entry, schema_file), name), file_index) in + declarations.into_iter().zip(names).zip(pairs) + { + seeds.push(SeedRecord { + name, + destination: None, + host: SeedHandles { + seed_file: seed_files[file_index].clone_ref(py), + schema_entry, + schema_file, + key: PyOnceLock::new(), + }, + }); + } + Ok(seeds) +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_reads/key.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/key.rs new file mode 100644 index 0000000000..ea4d396800 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/key.rs @@ -0,0 +1,15 @@ +//! The compiled source's `key`, as its Python attribute reads it. + +use crate::resource_views::_helpers::source_reads::native_source; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled source's `key`, as its Python attribute reads it. +pub fn key<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = source.py(); + if let Some(view) = native_source(source)? { + return view.key(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(source, "key") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_reads/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/mod.rs new file mode 100644 index 0000000000..92c5d37228 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/mod.rs @@ -0,0 +1,4 @@ +pub mod key; +pub mod name; +pub mod source_entry; +pub mod source_file; diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_reads/name.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/name.rs new file mode 100644 index 0000000000..1cb44fd30c --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/name.rs @@ -0,0 +1,15 @@ +//! The compiled source's `name`, as its Python attribute reads it. + +use crate::resource_views::_helpers::source_reads::native_source; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled source's `name`, as its Python attribute reads it. +pub fn name<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = source.py(); + if let Some(view) = native_source(source)? { + return view.name(py).and_then(|value| value.into_bound_py_any(py)); + } + native_field(source, "name") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_entry.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_entry.rs new file mode 100644 index 0000000000..b3cb95895b --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_entry.rs @@ -0,0 +1,17 @@ +//! The compiled source's `source_entry`, as its Python attribute reads it. + +use crate::resource_views::_helpers::source_reads::native_source; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled source's `source_entry`, as its Python attribute reads it. +pub fn source_entry<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = source.py(); + if let Some(view) = native_source(source)? { + return view + .source_entry(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(source, "source_entry") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_file.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_file.rs new file mode 100644 index 0000000000..f39b9ec4f8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_file.rs @@ -0,0 +1,17 @@ +//! The compiled source's `source_file`, as its Python attribute reads it. + +use crate::resource_views::_helpers::source_reads::native_source; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled source's `source_file`, as its Python attribute reads it. +pub fn source_file<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = source.py(); + if let Some(view) = native_source(source)? { + return view + .source_file(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(source, "source_file") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_views/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_views/mod.rs new file mode 100644 index 0000000000..5a8c6e25e0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_views/mod.rs @@ -0,0 +1,3 @@ +pub mod read_source_view; +pub mod replaced; +pub mod retain_sources; diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_views/read_source_view.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_views/read_source_view.rs new file mode 100644 index 0000000000..8b9956d343 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_views/read_source_view.rs @@ -0,0 +1,14 @@ +//! Read the source record behind a native source view, or `None` for any other object. + +use crate::resource_views::_helpers::source_views; +use crate::resource_views::types::SourceRecord; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; + +/// Read the source record behind a native source view, or `None` for any other object. +#[inline] +pub fn read_source_view<'py, T>( + view: &Bound<'py, PyAny>, + read: impl FnOnce(&SourceRecord, Python<'py>) -> PyResult, +) -> Option> { + source_views::read_source_view(view, read) +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_views/replaced.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_views/replaced.rs new file mode 100644 index 0000000000..f69e8e5220 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_views/replaced.rs @@ -0,0 +1,14 @@ +//! `dataclasses.replace(value, **changes)`. + +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; + +/// `dataclasses.replace(value, **changes)`. +pub fn replaced( + _py: Python<'_>, + value: &Bound<'_, PyAny>, + changes: &Bound<'_, PyDict>, +) -> PyResult> { + Ok(replaced_keywords(value, changes)?.unbind()) +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/source_views/retain_sources.rs b/crates/sqlbuild-python-project/src/resource_views/main/source_views/retain_sources.rs new file mode 100644 index 0000000000..379e29e8ec --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/source_views/retain_sources.rs @@ -0,0 +1,24 @@ +//! Keep `attached` in `owner`; return each source's view. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::resource_views::_helpers::source_views::source_record; +use crate::resource_views::models::NativeSourceInput; +use crate::resource_views::types::AttachedSource; +use pyo3::prelude::Bound; + +/// Keep `attached` in `owner`; return each source's view. +pub fn retain_sources( + owner: &Bound<'_, NativeCompiledProject>, + attached: Vec, +) -> Vec { + let mut project = owner.borrow_mut(); + let first: usize = project.sources().len(); + let count: usize = attached.len(); + for source in attached { + project.retain_source(source_record(source)); + } + drop(project); + (first..first + count) + .map(|index| NativeSourceInput::new(owner.clone().unbind(), index)) + .collect() +} diff --git a/crates/sqlbuild-python-project/src/resource_views/mod.rs b/crates/sqlbuild-python-project/src/resource_views/mod.rs new file mode 100644 index 0000000000..d68d2e3db3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/mod.rs @@ -0,0 +1,6 @@ +//! Python views of compiled audits, functions, seeds and sources, and their reads. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/resource_views/models.rs b/crates/sqlbuild-python-project/src/resource_views/models.rs new file mode 100644 index 0000000000..a0afc09f48 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/models.rs @@ -0,0 +1,653 @@ +//! Public records of resource views. + +use crate::compiled_project::models::NativeCompiledProject; +use crate::compiled_views::main::compiled_models::defaulted_destination::defaulted_destination; +use crate::compiled_views::main::compiled_models::object_key::object_key; +use crate::compiled_views::main::compiled_models::relation_location::relation_location; +use crate::compiled_views::main::model_inputs::view_classes::view_classes; +use crate::compiled_views::models::InputViewClasses; +use crate::graph::models::ResourceRow; +use crate::resource_views::_helpers::function_views::NOT_ASSEMBLED; +use crate::resource_views::_helpers::function_views::read_function; +use crate::resource_views::_helpers::seed_views::SEED_RESOURCE; +use crate::resource_views::_helpers::seed_views::TAGS_ATTRIBUTE; +use crate::resource_views::_helpers::seed_views::read_seed; +use crate::resource_views::_helpers::source_views::SOURCE_RESOURCE; +use crate::resource_views::_helpers::source_views::SourceAssemblyRow; +use crate::resource_views::_helpers::source_views::read_source; +use crate::resource_views::types::AttachedSource; +use crate::resource_views::types::SourceViews; +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyTuple; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; +use sqlbuild_project::models::DestinationFact; +use sqlbuild_project::types::ObjectKeyRow; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// An attached audit's user values. +#[derive(Debug)] +pub struct AuditHandles { + /// The discovered (or built-in) audit file and block the SQL comes from. + pub audit_file: Py, + pub audit_block: Py, + /// The authored measurement policy specs, carrying user values. + pub thresholds: Py, + pub minimum_samples: Py, + pub evidence_limit: Py, + /// Usage records, the identities the scope index is built from. + pub usages: Py, +} + +/// An audit record's Python projections, each built once on first access. +pub struct AuditProjections { + pub references: PyOnceLock>, + pub key: PyOnceLock>, +} + +/// One attached audit, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeAuditInput { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeAuditInput { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +/// One assembled audit, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeCompiledAudit { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeCompiledAudit { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +impl NativeCompiledAudit { + /// The `CompiledAudit` field values, in field order, without building the record. + pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.scope_deps(py)?.into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.definition_name(py)?.into_bound_py_any(py)?, + self.audit_file(py)?.into_bound(py), + self.audit_block(py)?.into_bound(py), + self.sql_body(py)?.into_bound_py_any(py)?, + self.evaluation_mode(py)?.into_bound(py), + self.measurement_contract(py)?.into_bound(py), + self.thresholds(py)?.into_bound(py), + self.minimum_samples(py)?.into_bound(py), + self.measure_sql(py)?.into_bound_py_any(py)?, + self.evidence_sql(py)?.into_bound_py_any(py)?, + self.evidence_limit(py)?.into_bound(py), + self.references(py)?.into_bound(py).into_any(), + self.attached_target_kind(py)?, + self.attached_target_name(py)?.into_bound_py_any(py)?, + self.attached_column_name(py)?.into_bound_py_any(py)?, + self.severity(py)?, + self.run_scope(py)?.into_bound_py_any(py)?, + pyo3::types::PyBool::new(py, self.always_run(py)?) + .to_owned() + .into_any(), + self.description(py)?.into_bound_py_any(py)?, + self.meta(py)?.into_bound(py), + ]) + } +} + +/// An attached function's user values. +#[derive(Debug)] +pub struct FunctionHandles { + /// The discovered function file. + pub function_file: Py, + /// Usage records, the identities the scope index is built from. + pub usages: Py, +} + +/// A function record's Python projections, each built once on first access. +pub struct FunctionProjections { + pub(crate) arguments: PyOnceLock>, + pub(crate) return_columns: PyOnceLock>, + pub references: PyOnceLock>, + pub key: PyOnceLock>, +} + +/// One attached function, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeFunctionInput { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeFunctionInput { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +/// One assembled function, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub struct NativeCompiledFunction { + pub(crate) owner: Py, + pub(crate) index: usize, + /// The destination and fingerprint destination with target defaults applied. + pub(crate) defaulted: Option<(DestinationFact, DestinationFact)>, + pub(crate) destination: PyOnceLock>, + pub(crate) fingerprint_destination: PyOnceLock>, +} + +impl std::fmt::Debug for NativeCompiledFunction { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("NativeCompiledFunction") + .finish_non_exhaustive() + } +} + +impl NativeCompiledFunction { + pub fn new(owner: Py, index: usize) -> Self { + Self::defaulted(owner, index, None) + } + + /// The function's project graph row: its key, the resources it reads and its tags. + pub fn graph_row(&self, py: Python<'_>) -> PyResult { + read_function(&self.owner, self.index, py, |function, _| { + let deps: &[ObjectKeyRow] = function + .deps + .as_deref() + .ok_or_else(|| PyRuntimeError::new_err(NOT_ASSEMBLED))?; + Ok(ResourceRow { + key: ( + function.attached.node_type().to_owned(), + function.attached.name.clone(), + ), + deps: deps.to_vec(), + tags: function.attached.tags.clone(), + folder: None, + }) + }) + } + + pub(crate) fn defaulted( + owner: Py, + index: usize, + defaulted: Option<(DestinationFact, DestinationFact)>, + ) -> Self { + Self { + owner, + index, + defaulted, + destination: PyOnceLock::new(), + fingerprint_destination: PyOnceLock::new(), + } + } + + /// The destination and fingerprint destination, with target defaults once applied. + pub fn current_destinations( + &self, + py: Python<'_>, + ) -> PyResult<(DestinationFact, DestinationFact)> { + if let Some(defaulted) = &self.defaulted { + return Ok(defaulted.clone()); + } + read_function(&self.owner, self.index, py, |function, _| { + Ok(( + function.attached.destination.clone(), + function.attached.fingerprint_destination.clone(), + )) + }) + } +} + +impl NativeCompiledFunction { + /// The `CompiledFunction` field values, in field order, without building the record. + pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.deps(py)?.into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.relative_path(py)?.into_bound(py), + self.arguments(py)?.into_bound(py).into_any(), + self.returns(py)?.into_bound_py_any(py)?, + self.body_sql(py)?.into_bound_py_any(py)?, + self.destination(py)?.into_bound(py), + self.fingerprint_destination(py)?.into_bound(py), + self.return_columns(py)?.into_bound(py).into_any(), + self.references(py)?.into_bound(py).into_any(), + self.language(py)?, + self.source_file_path(py)?.into_bound(py), + self.runtime_version(py)?.into_bound_py_any(py)?, + self.entry_point(py)?.into_bound_py_any(py)?, + self.packages(py)?.into_any(), + self.tags(py)?.into_any(), + self.description(py)?.into_bound_py_any(py)?, + ]) + } +} + +/// `(database, schema, logical database, logical schema)` of a seed's destination. +#[derive(Debug)] +pub struct SeedNamespace { + pub database: Option, + pub schema: Option, + pub logical_database: Option, + pub logical_schema: Option, +} + +impl<'py> pyo3::FromPyObject<'py> for SeedNamespace { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let (database, schema, logical_database, logical_schema): ( + Option, + Option, + Option, + Option, + ) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { + database, + schema, + logical_database, + logical_schema, + }) + } +} + +/// A seed's name, CSV file, declaration and declaring schema file. +#[derive(Debug)] +pub struct SeedParts<'a> { + pub name: &'a str, + pub seed_file: &'a Py, + pub schema_entry: &'a Py, + pub schema_file: &'a Py, +} + +/// A seed's discovered objects and the key its views build. +pub struct SeedHandles { + /// The discovered CSV file. + pub(crate) seed_file: Py, + /// The discovered declaration, whose spec carries user-authored column metadata. + pub(crate) schema_entry: Py, + /// The schema file declaring the seed. + pub(crate) schema_file: Py, + pub(crate) key: PyOnceLock>, +} + +/// One attached seed, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeSeedInput { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeSeedInput { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +#[pymethods] +impl NativeSeedInput { + #[getter] + pub fn seed_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.seed_file.clone_ref(py)) + }) + } + + #[getter] + pub fn schema_entry(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.schema_entry.clone_ref(py)) + }) + } + + #[getter] + pub fn schema_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.schema_file.clone_ref(py)) + }) + } +} + +/// One assembled seed, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub struct NativeCompiledSeed { + pub(crate) owner: Py, + pub(crate) index: usize, + /// The destination with target defaults applied, replacing the assembled one. + pub(crate) defaulted: Option, + pub(crate) destination: PyOnceLock>, +} + +impl std::fmt::Debug for NativeCompiledSeed { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("NativeCompiledSeed") + .finish_non_exhaustive() + } +} + +impl NativeCompiledSeed { + pub fn new(owner: Py, index: usize) -> Self { + Self { + owner, + index, + defaulted: None, + destination: PyOnceLock::new(), + } + } + + /// The seed's project graph row: its key and its declared tags. + pub fn graph_row(&self, py: Python<'_>) -> PyResult { + read_seed(&self.owner, self.index, py, |seed, py| { + let mut tags: Vec = Vec::new(); + for tag in native_field(seed.host.schema_entry.bind(py), TAGS_ATTRIBUTE)?.try_iter()? { + tags.push(tag?.str()?.extract()?); + } + Ok(ResourceRow { + key: (SEED_RESOURCE.to_owned(), seed.name.clone()), + deps: Vec::new(), + tags, + folder: None, + }) + }) + } + + pub fn current_destination(&self, py: Python<'_>) -> PyResult { + if let Some(defaulted) = &self.defaulted { + return Ok(defaulted.clone()); + } + read_seed(&self.owner, self.index, py, |seed, _| { + seed.destination + .clone() + .ok_or_else(|| PyRuntimeError::new_err("the seed was not assembled")) + }) + } +} + +#[pymethods] +impl NativeCompiledSeed { + /// The seed's planner-facing `CompiledObjectKey`. + #[getter] + pub fn key(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + read_seed(&self.owner, self.index, py, |seed, py| { + seed.host + .key + .get_or_try_init(py, || { + object_key(&classes.compiled, SEED_RESOURCE, &seed.name) + }) + .map(|key| key.clone_ref(py)) + }) + } + + #[getter] + pub fn deps<'py>(&self, py: Python<'py>) -> Bound<'py, PyTuple> { + PyTuple::empty(py) + } + + #[getter] + pub fn name(&self, py: Python<'_>) -> PyResult { + read_seed(&self.owner, self.index, py, |seed, _| Ok(seed.name.clone())) + } + + #[getter] + pub fn seed_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.seed_file.clone_ref(py)) + }) + } + + #[getter] + pub fn schema_entry(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.schema_entry.clone_ref(py)) + }) + } + + #[getter] + pub fn schema_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.host.schema_file.clone_ref(py)) + }) + } + + /// The seed's planner-facing `CompiledRelationLocation`. + #[getter] + pub fn destination(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.destination + .get_or_try_init(py, || { + relation_location(&classes.compiled, &self.current_destination(py)?) + }) + .map(|destination| destination.clone_ref(py)) + } + + /// This seed with the adapter's default schema and database filled into its destination. + pub fn with_target_defaults( + &self, + py: Python<'_>, + defaults: (Option, Option), + render: Bound<'_, PyAny>, + ) -> PyResult { + let defaulted: DestinationFact = + defaulted_destination(&self.current_destination(py)?, defaults, &render)?; + Ok(Self { + owner: self.owner.clone_ref(py), + index: self.index, + defaulted: Some(defaulted), + destination: PyOnceLock::new(), + }) + } + + #[getter(external)] + pub fn is_external(&self) -> bool { + false + } +} + +impl NativeCompiledSeed { + /// The `CompiledSeed` field values, in field order, without building the record. + pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.deps(py).into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.seed_file(py)?.into_bound(py), + self.schema_entry(py)?.into_bound(py), + self.schema_file(py)?.into_bound(py), + self.destination(py)?.into_bound(py), + pyo3::types::PyBool::new(py, self.is_external()) + .to_owned() + .into_any(), + ]) + } +} + +/// An attached source's user values. +#[derive(Debug)] +pub struct SourceHandles { + /// The discovered source file. + pub source_file: Py, + /// The discovered entry; its spec carries user-authored metadata. + pub entry: Py, + /// Template-expanded user metadata. + pub meta: Py, + /// Template-expanded column specs, which carry user metadata and audit arguments. + pub columns: Py, + /// Template-expanded table audit specs, which carry user audit arguments. + pub audits: Py, + /// Usage records, the identities the scope index is built from. + pub usages: Py, +} + +/// What a source declares about its keys: its column facts and `(audit, column argument)` rows. +#[derive(Debug)] +pub struct SourceKeyFacts { + pub columns: Vec, + pub audits: Vec<(String, Option)>, +} + +/// A source record's key facts and entries, each built once from its Python values. +pub struct SourceProjections { + pub(crate) key_facts: PyOnceLock, + pub(crate) attached_entry: PyOnceLock>, + pub(crate) assembled_entry: PyOnceLock>, + pub key: PyOnceLock>, +} + +/// One attached source, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeSourceInput { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeSourceInput { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +#[pymethods] +impl NativeSourceInput { + #[getter] + pub fn source_entry(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + source.attached_entry(py) + }) + } + + #[getter] + pub fn source_file(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + Ok(source.attached.host.source_file.clone_ref(py)) + }) + } + + #[getter] + pub fn declaration_usages(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + Ok(source.attached.host.usages.clone_ref(py)) + }) + } + + /// `(name, has a loader, database, schema)`, what project assembly resolves a namespace from. + #[getter] + pub(crate) fn assembly_row(&self, py: Python<'_>) -> PyResult { + read_source(&self.owner, self.index, py, |source, py| { + let attached: &AttachedSource = &source.attached; + Ok(SourceAssemblyRow { + name: attached.name.clone(), + managed: !native_field(attached.host.entry.bind(py), "loader")?.is_none(), + database: attached.database.clone(), + schema: attached.schema.clone(), + }) + }) + } +} + +/// One assembled source, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct NativeCompiledSource { + pub(crate) owner: Py, + pub(crate) index: usize, +} + +impl NativeCompiledSource { + pub fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } + + /// The source's project graph row: its key, with no dependencies, tags or folder. + pub fn graph_row(&self, py: Python<'_>) -> PyResult { + read_source(&self.owner, self.index, py, |source, _| { + Ok(ResourceRow { + key: (SOURCE_RESOURCE.to_owned(), source.attached.name.clone()), + deps: Vec::new(), + tags: Vec::new(), + folder: None, + }) + }) + } +} + +#[pymethods] +impl NativeCompiledSource { + /// The source's planner-facing `CompiledObjectKey`. + #[getter] + pub fn key(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + read_source(&self.owner, self.index, py, |source, py| { + source + .host + .key + .get_or_try_init(py, || { + object_key(&classes.compiled, SOURCE_RESOURCE, &source.attached.name) + }) + .map(|key| key.clone_ref(py)) + }) + } + + #[getter] + pub fn deps<'py>(&self, py: Python<'py>) -> Bound<'py, PyTuple> { + PyTuple::empty(py) + } + + #[getter] + pub fn name(&self, py: Python<'_>) -> PyResult { + read_source(&self.owner, self.index, py, |source, _| { + Ok(source.attached.name.clone()) + }) + } + + #[getter] + pub fn source_entry(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + source.assembled_entry(py) + }) + } + + #[getter] + pub fn source_file(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + Ok(source.attached.host.source_file.clone_ref(py)) + }) + } +} + +impl NativeCompiledSource { + /// The `CompiledSource` field values, in field order, without building the record. + pub fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.deps(py).into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.source_entry(py)?.into_bound(py), + self.source_file(py)?.into_bound(py), + ]) + } +} diff --git a/crates/sqlbuild-python-project/src/resource_views/types.rs b/crates/sqlbuild-python-project/src/resource_views/types.rs new file mode 100644 index 0000000000..2a583e5a18 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/types.rs @@ -0,0 +1,53 @@ +//! Public traits and type aliases of resource views. + +use crate::compiled_views::models::PyProjectHost; +use crate::resource_views::models::SeedNamespace; +use crate::resource_views::models::SeedParts; +use crate::resource_views::models::SourceKeyFacts; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use sqlbuild_project::models as project_models; + +/// What the audit loop attached for one singular audit block or attached generic audit. +pub type AttachedAudit = project_models::AttachedAudit; + +/// One attached audit held by the compiled project. +pub type AuditRecord = project_models::AuditRecord; + +/// What the function loop attached for one SQL or Python function. +pub type AttachedFunction = project_models::AttachedFunction; + +/// One attached function held by the compiled project. +pub type FunctionRecord = project_models::FunctionRecord; + +/// One seed CSV paired with its declaration; the discovered objects are discovery's. +pub type SeedRecord = project_models::SeedRecord; + +/// What a seed record reads and keeps through its Python values. +pub trait SeedViews { + /// The seed's name, CSV file, declaration and declaring schema file. + fn parts(&self) -> SeedParts<'_>; + + /// Keep the destination native project assembly resolved for the seed. + fn set_namespace(&mut self, namespace: SeedNamespace); +} + +/// What the source loop attached for one source: its expanded metadata and usages. +pub type AttachedSource = project_models::AttachedSource; + +/// One attached source held by the compiled project. +pub type SourceRecord = project_models::SourceRecord; + +/// What a source record projects from its Python values, each built once. +pub trait SourceViews { + /// The source's declared column facts and audit key rows, read once from its expanded specs. + fn key_facts(&self, py: Python<'_>) -> PyResult<&SourceKeyFacts>; + + /// The attached `SourceEntry`: the discovered entry with its expanded metadata. + fn attached_entry(&self, py: Python<'_>) -> PyResult>; + + /// The assembled `SourceEntry`: the attached entry in its resolved namespace. + fn assembled_entry(&self, py: Python<'_>) -> PyResult>; +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs similarity index 89% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs rename to crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs index feeeae8e32..9b391baf7d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs @@ -2,13 +2,14 @@ use std::collections::BTreeSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compiled_views::model_reads::{inferred_rows, star_flags}; -use crate::bindings::_helpers::semantic_shapes::delta_shapes::semantic_reasons; -use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; +use crate::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use crate::compiled_views::main::compiled_models::star_flags::star_flags; +use crate::semantic_shapes::_helpers::delta_shapes::semantic_reasons; +use crate::semantic_shapes::_helpers::shapes::project_shapes; use sqlbuild_project::types::InferredRow; use sqlbuild_python_base::compile_context::models::ModelCoverage; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -110,7 +111,3 @@ fn own_reasons<'py>( } Ok(partial) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(semantic_coverage, module)?) -} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/declarations.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/declarations.rs new file mode 100644 index 0000000000..e203d01972 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/declarations.rs @@ -0,0 +1,3 @@ +//! The function names and declared types native binding treats as known. + +pub(crate) const UNKNOWN: &str = "UNKNOWN"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/delta_shapes.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/delta_shapes.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/delta_shapes.rs rename to crates/sqlbuild-python-project/src/semantic_shapes/_helpers/delta_shapes.rs diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/mod.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/mod.rs new file mode 100644 index 0000000000..5131e6afd5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod coverage; +pub(crate) mod declarations; +pub(crate) mod delta_shapes; +pub(crate) mod shapes; +pub(crate) mod source_bindings; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs similarity index 85% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs rename to crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs index 4063086085..4d3c222ee1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs @@ -1,37 +1,34 @@ //! Closed relation shapes of a compiled project, as binding and semantic checks read them. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::semantic_shapes::main::snapshot_validity_columns::snapshot_validity_columns; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::analysis::binding_catalog::{ - BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, CatalogKey, -}; -use crate::bindings::_helpers::analysis::normalization::normalize_analysis_sql; -use crate::bindings::_helpers::analysis_session::stage_functions::infer_native_expression_source_shapes; -use crate::bindings::_helpers::compiled_project::rows::ReferenceTuple; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - DeclaredRow, NativeCompiledModel, -}; -use crate::bindings::_helpers::compiled_views::model_reads::{ - declared_rows, dynamic_columns_declared, inferred_rows, star_flags, unchecked_columns, -}; +use crate::analysis::constants::CASE_SENSITIVE_BINDING_DIALECTS; +use crate::analysis::main::normalize_analysis_sql::normalize_analysis_sql; +use crate::analysis::models::{BindingCatalog, CatalogKey}; +use crate::analysis_session::main::infer_native_expression_source_shapes::infer_native_expression_source_shapes; +use crate::compiled_project::models::ReferenceTuple; +use crate::compiled_views::main::compiled_models::declared_rows::declared_rows; +use crate::compiled_views::main::compiled_models::dynamic_columns_declared::dynamic_columns_declared; +use crate::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use crate::compiled_views::main::compiled_models::star_flags::star_flags; +use crate::compiled_views::main::compiled_models::unchecked_columns::unchecked_columns; +use crate::compiled_views::models::NativeCompiledModel; +use crate::compiled_views::types::DeclaredRow; use sqlbuild_project::types::InferredRow; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; const UNKNOWN: &str = "UNKNOWN"; const GENERIC_DIALECT: &str = "generic"; -/// Snapshot validity start column when the model names none. -pub(crate) const VALID_FROM_DEFAULT: &str = "valid_from"; -/// Snapshot validity end column when the model names none. -pub(crate) const VALID_TO_DEFAULT: &str = "valid_to"; + const TABLE_FUNCTION_PREFIX: &str = "__sqlbuild_table_function_"; const UDF_KIND: &str = "udf"; const TABLE_FUNCTION_KIND: &str = "table_fn"; @@ -115,29 +112,6 @@ pub(crate) fn published_shape<'py>( Ok(shape) } -/// A snapshot model's validity column names, which the snapshot adds to its query's columns. -pub(crate) fn snapshot_validity_columns(model: &Bound<'_, PyAny>) -> PyResult> { - let py: Python<'_> = model.py(); - let values: Bound<'_, PyAny> = native_field(&native_field(model, "config")?, "values")?; - let snapshot: Bound<'_, PyAny> = enum_member(py, RecordId::MaterializationType, "SNAPSHOT")?; - if !mapping_get(&values, "materialized")?.eq(snapshot)? { - return Ok(Vec::new()); - } - let mut names: Vec = Vec::with_capacity(2); - for (key, default) in [ - ("valid_from_column", VALID_FROM_DEFAULT), - ("valid_to_column", VALID_TO_DEFAULT), - ] { - let name: Bound<'_, PyAny> = mapping_get(&values, key)?; - names.push(if name.is_truthy()? { - name.str()?.extract()? - } else { - default.to_owned() - }); - } - Ok(names) -} - /// Inferred columns, re-quoted from authored SQL where the dialect keeps quoted case. fn binding_shape<'py>( model: &Bound<'py, PyAny>, @@ -327,9 +301,7 @@ fn model_shapes<'py>( let mut pending: Vec> = native_field(project, "models")? .try_iter()? .collect::>()?; - crate::bindings::_helpers::semantic_shapes::delta_shapes::inject_model_shapes( - &pending, shapes, - )?; + crate::semantic_shapes::_helpers::delta_shapes::inject_model_shapes(&pending, shapes)?; while !pending.is_empty() { let mut remaining: Vec> = Vec::new(); for model in &pending { @@ -375,7 +347,7 @@ fn model_shapes<'py>( } pending = remaining; } - crate::bindings::_helpers::semantic_shapes::delta_shapes::record_model_shapes(project, shapes) + crate::semantic_shapes::_helpers::delta_shapes::record_model_shapes(project, shapes) } fn all_known(names: &[String], shapes: &Bound<'_, PyDict>) -> PyResult { @@ -390,14 +362,16 @@ fn all_known(names: &[String], shapes: &Bound<'_, PyDict>) -> PyResult { /// Closed relation shapes; `profile` defaults to the project's dialect. #[pyfunction] #[pyo3(signature = (**request))] -fn semantic_shapes<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn semantic_shapes<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("semantic_shapes", request)?; project_shapes(&request.get("project")?, request.optional("profile")?) } /// The analysis names the references bind, UDFs excluded. #[pyfunction] -fn binding_relation_names<'py>( +pub(crate) fn binding_relation_names<'py>( references: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = references.py(); @@ -422,7 +396,7 @@ fn binding_relation_names<'py>( /// A model relation's published shape: `published_model_shape(*, model, columns, inputs, profile)`. #[pyfunction] #[pyo3(signature = (**request))] -fn published_model_shape<'py>( +pub(crate) fn published_model_shape<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("published_model_shape", request)?; @@ -433,9 +407,3 @@ fn published_model_shape<'py>( &request.get("profile")?, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(semantic_shapes, module)?)?; - module.add_function(wrap_pyfunction!(published_model_shape, module)?)?; - module.add_function(wrap_pyfunction!(binding_relation_names, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs rename to crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs index 0cc5d322ae..1b4a796e94 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs @@ -1,24 +1,21 @@ //! Rebind selected models to source columns the planner already inspected. +use crate::semantic_shapes::main::declared_types::declared_types; +use crate::semantic_shapes::main::function_names::function_names; use std::collections::HashSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::analysis::binding_catalog::{ - self, CASE_SENSITIVE_BINDING_DIALECTS, NATIVE_DIALECT_ALIASES, schema_validations, -}; -use crate::bindings::_helpers::analysis::schema_request::{ - SchemaRequestParts, schema_validation_request, -}; -use crate::bindings::_helpers::analysis_session::stage_functions::infer_native_query_columns; -use crate::bindings::_helpers::semantic_shapes::constants::SOURCE_REBINDING_ANALYSES; -use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; -use crate::bindings::_helpers::semantic_shapes::shapes::{ - model_inputs, project_shapes, published_shape, -}; -use crate::bindings::models::BindingPositions; +use crate::analysis::constants::{CASE_SENSITIVE_BINDING_DIALECTS, NATIVE_DIALECT_ALIASES}; +use crate::analysis::main::schema_validation_request::schema_validation_request; +use crate::analysis::main::schema_validations::schema_validations; +use crate::analysis::models::BindingPositions; +use crate::analysis::models::SchemaRequestParts; +use crate::analysis_session::main::infer_native_query_columns::infer_native_query_columns; +use crate::semantic_shapes::_helpers::shapes::{model_inputs, project_shapes, published_shape}; +use crate::semantic_shapes::constants::SOURCE_REBINDING_ANALYSES; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; @@ -281,7 +278,7 @@ fn validated_diagnostics<'py>( let known_functions: Object<'py> = PyTuple::new(py, function_names(&functions)?)?.into_any(); let known_types: Object<'py> = PyTuple::new(py, declared_types(&functions, shapes.as_any())?)?.into_any(); - let ignores: bool = binding_catalog::ignores_quoted_case( + let ignores: bool = crate::analysis::main::ignores_quoted_case::ignores_quoted_case( &native_field(project, "effective_connection")?, dialect, )?; @@ -474,7 +471,7 @@ fn is_folded_identifier(name: &str) -> bool { } fn ignores_quoted_case<'py>(project: &Object<'py>, dialect: &Object<'py>) -> PyResult> { - let ignores: bool = binding_catalog::ignores_quoted_case( + let ignores: bool = crate::analysis::main::ignores_quoted_case::ignores_quoted_case( &native_field(project, "effective_connection")?, dialect, )?; @@ -497,7 +494,3 @@ fn call<'py>(callable: &Object<'py>, pairs: &[(&str, Object<'py>)]) -> PyResult< } callable.call((), Some(&arguments)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(source_binding_diagnostics, module)?) -} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/constants.rs b/crates/sqlbuild-python-project/src/semantic_shapes/constants.rs new file mode 100644 index 0000000000..98eac192ad --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/constants.rs @@ -0,0 +1,10 @@ +//! Public constants of semantic shapes. + +/// Model queries analysed again against inspected source columns. +pub(crate) const SOURCE_REBINDING_ANALYSES: &str = "source_rebinding_analyses"; + +/// Snapshot validity start column when the model names none. +pub const VALID_FROM_DEFAULT: &str = "valid_from"; + +/// Snapshot validity end column when the model names none. +pub const VALID_TO_DEFAULT: &str = "valid_to"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/declared_types.rs similarity index 66% rename from crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs rename to crates/sqlbuild-python-project/src/semantic_shapes/main/declared_types.rs index 28c19328b7..435e8830ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/declared_types.rs @@ -1,27 +1,13 @@ -//! The function names and declared types native binding treats as known. - -use std::collections::BTreeSet; +//! Types named by relation shapes and function signatures, sorted, `UNKNOWN` left out. +use crate::semantic_shapes::_helpers::declarations::UNKNOWN; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; - -const UNKNOWN: &str = "UNKNOWN"; - -/// Each declared function's name and its UDF and table-function analysis names, sorted. -pub(crate) fn function_names(functions: &Bound<'_, PyAny>) -> PyResult> { - let mut names: BTreeSet = BTreeSet::new(); - for function in functions.try_iter()? { - let name: String = native_field(&function?, "name")?.extract()?; - names.insert(format!("__sqlbuild_udf_{name}")); - names.insert(format!("__sqlbuild_table_function_{name}")); - names.insert(name); - } - Ok(names.into_iter().collect()) -} +use std::collections::BTreeSet; /// Types named by relation shapes and function signatures, sorted, `UNKNOWN` left out. -pub(crate) fn declared_types( +pub fn declared_types( functions: &Bound<'_, PyAny>, column_types: &Bound<'_, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/function_names.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/function_names.rs new file mode 100644 index 0000000000..6270554ccd --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/function_names.rs @@ -0,0 +1,17 @@ +//! Each declared function's name and its UDF and table-function analysis names, sorted. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::BTreeSet; + +/// Each declared function's name and its UDF and table-function analysis names, sorted. +pub fn function_names(functions: &Bound<'_, PyAny>) -> PyResult> { + let mut names: BTreeSet = BTreeSet::new(); + for function in functions.try_iter()? { + let name: String = native_field(&function?, "name")?.extract()?; + names.insert(format!("__sqlbuild_udf_{name}")); + names.insert(format!("__sqlbuild_table_function_{name}")); + names.insert(name); + } + Ok(names.into_iter().collect()) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/mod.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/mod.rs new file mode 100644 index 0000000000..f55fb1fa45 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/mod.rs @@ -0,0 +1,9 @@ +pub mod declared_types; +pub mod function_names; +pub mod project_shapes; +pub mod register_coverage; +pub mod register_shapes; +pub mod register_source_bindings; +pub mod semantic_coverage; +pub mod snapshot_validity_columns; +pub mod source_binding_diagnostics; diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/project_shapes.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/project_shapes.rs new file mode 100644 index 0000000000..11b86cb4a5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/project_shapes.rs @@ -0,0 +1,14 @@ +//! Closed shapes by relation name; absence means open, never an empty physical table. + +use crate::semantic_shapes::_helpers::shapes; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyDict; + +/// Closed shapes by relation name; absence means open, never an empty physical table. +#[inline] +pub fn project_shapes<'py>( + project: &Bound<'py, PyAny>, + profile: Option>, +) -> PyResult> { + shapes::project_shapes(project, profile) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/register_coverage.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_coverage.rs new file mode 100644 index 0000000000..a478031553 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_coverage.rs @@ -0,0 +1,10 @@ +//! Register the coverage functions and classes. + +use crate::semantic_shapes::_helpers::coverage::semantic_coverage; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the coverage functions and classes. +pub fn register_coverage(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(semantic_coverage, module)?) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs new file mode 100644 index 0000000000..799c2deae1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs @@ -0,0 +1,14 @@ +//! Register the shapes functions and classes. + +use crate::semantic_shapes::_helpers::shapes::binding_relation_names; +use crate::semantic_shapes::_helpers::shapes::published_model_shape; +use crate::semantic_shapes::_helpers::shapes::semantic_shapes; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the shapes functions and classes. +pub fn register_shapes(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(semantic_shapes, module)?)?; + module.add_function(wrap_pyfunction!(published_model_shape, module)?)?; + module.add_function(wrap_pyfunction!(binding_relation_names, module)?) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/register_source_bindings.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_source_bindings.rs new file mode 100644 index 0000000000..045414aef1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/register_source_bindings.rs @@ -0,0 +1,10 @@ +//! Register the source bindings functions and classes. + +use crate::semantic_shapes::_helpers::source_bindings::source_binding_diagnostics; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the source bindings functions and classes. +pub fn register_source_bindings(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(source_binding_diagnostics, module)?) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/semantic_coverage.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/semantic_coverage.rs new file mode 100644 index 0000000000..c9e63abd2d --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/semantic_coverage.rs @@ -0,0 +1,11 @@ +//! Partial-check reasons by model name; empty when SQL analysis is off. + +use crate::semantic_shapes::_helpers::coverage; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyDict; + +/// Partial-check reasons by model name; empty when SQL analysis is off. +#[inline] +pub fn semantic_coverage<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + coverage::semantic_coverage(project) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/snapshot_validity_columns.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/snapshot_validity_columns.rs new file mode 100644 index 0000000000..3f3105ad01 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/snapshot_validity_columns.rs @@ -0,0 +1,32 @@ +//! A snapshot model's validity column names, which the snapshot adds to its query's columns. + +use crate::semantic_shapes::constants::VALID_FROM_DEFAULT; +use crate::semantic_shapes::constants::VALID_TO_DEFAULT; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_base::records::types::RecordId; + +/// A snapshot model's validity column names, which the snapshot adds to its query's columns. +pub fn snapshot_validity_columns(model: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = model.py(); + let values: Bound<'_, PyAny> = native_field(&native_field(model, "config")?, "values")?; + let snapshot: Bound<'_, PyAny> = enum_member(py, RecordId::MaterializationType, "SNAPSHOT")?; + if !mapping_get(&values, "materialized")?.eq(snapshot)? { + return Ok(Vec::new()); + } + let mut names: Vec = Vec::with_capacity(2); + for (key, default) in [ + ("valid_from_column", VALID_FROM_DEFAULT), + ("valid_to_column", VALID_TO_DEFAULT), + ] { + let name: Bound<'_, PyAny> = mapping_get(&values, key)?; + names.push(if name.is_truthy()? { + name.str()?.extract()? + } else { + default.to_owned() + }); + } + Ok(names) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/main/source_binding_diagnostics.rs b/crates/sqlbuild-python-project/src/semantic_shapes/main/source_binding_diagnostics.rs new file mode 100644 index 0000000000..aee9fd01dc --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/main/source_binding_diagnostics.rs @@ -0,0 +1,13 @@ +//! `source_binding_diagnostics(*, project, columns, profile, selected_keys)`. + +use crate::semantic_shapes::_helpers::source_bindings; +use pyo3::prelude::{Bound, PyResult}; +use pyo3::types::{PyDict, PyTuple}; + +/// `source_binding_diagnostics(*, project, columns, profile, selected_keys)`. +#[inline] +pub fn source_binding_diagnostics<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + source_bindings::source_binding_diagnostics(request) +} diff --git a/crates/sqlbuild-python-project/src/semantic_shapes/mod.rs b/crates/sqlbuild-python-project/src/semantic_shapes/mod.rs new file mode 100644 index 0000000000..227cc415d8 --- /dev/null +++ b/crates/sqlbuild-python-project/src/semantic_shapes/mod.rs @@ -0,0 +1,5 @@ +//! Semantic shapes: declarations, coverage, delta shapes and source bindings. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs index b9ab6050b2..067d5c84bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs @@ -8,8 +8,9 @@ use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; use sqlbuild_cursor::algebra::models::{Grain, Scalar}; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::sql_test_glue::constants::{ +use crate::compiled_views::main::compiled_models::name_object::name_object; +use crate::compiled_views::main::compiled_models::query_sql::query_sql; +use crate::sql_test_glue::constants::{ CURSOR_ALGEBRA_EXCEPTIONS, CURSOR_END_KEY, CURSOR_START_KEY, DEFAULT_INTEGER_END, DEFAULT_INTEGER_START, DEFAULT_TIMESTAMP_END, DEFAULT_TIMESTAMP_START, INTEGER_CURSOR_TYPE, TIMESTAMP_CURSOR_TYPE, @@ -44,19 +45,19 @@ struct BoundRequest<'a, 'py> { } /// Whether SQL calls `__cursor_start()` or `__cursor_end()`. -pub(super) fn uses_cursor_intrinsics(py: Python<'_>, sql: &str) -> PyResult { +pub(crate) fn uses_cursor_intrinsics(py: Python<'_>, sql: &str) -> PyResult { Ok(resolve_cursor_intrinsics(py, sql, None, None)?.1) } /// Whether a SQL test header declares its own cursor window. -pub(super) fn declares_cursor_window(test: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn declares_cursor_window(test: &Bound<'_, PyAny>) -> PyResult { let block = native_field(test, "test_block")?; Ok(!native_field(&block, "cursor_start")?.is_none() || !native_field(&block, "cursor_end")?.is_none()) } /// SQL with intrinsics rendered from the test's declared window, or the wide default window. -pub(super) fn render_test_cursor_intrinsics<'py>( +pub(crate) fn render_test_cursor_intrinsics<'py>( sql: Bound<'py, PyAny>, model: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, @@ -85,14 +86,14 @@ pub(super) fn render_test_cursor_intrinsics<'py>( } /// Chain models using intrinsics, after validating the test's declared window against each. -pub(super) fn declared_window_cursor_models<'py>( +pub(crate) fn declared_window_cursor_models<'py>( test: &Bound<'py, PyAny>, chain_models: Vec>, ) -> PyResult>> { let py = test.py(); let mut cursor_models: Vec> = Vec::new(); for model in chain_models { - if uses_cursor_intrinsics(py, &model_reads::query_sql(&model)?.extract::()?)? { + if uses_cursor_intrinsics(py, &query_sql(&model)?.extract::()?)? { cursor_models.push(model); } } @@ -263,7 +264,7 @@ fn model_cursor<'py>(model: &Bound<'py, PyAny>) -> PyResult> { _ => None, }; Ok(ModelCursor { - name: model_reads::name_object(model)?.extract()?, + name: name_object(model)?.extract()?, cursor_type, type_name, grain, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/fixture_validation.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/fixture_validation.rs index a394bd21d6..d32a0a8fd2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/fixture_validation.rs @@ -8,14 +8,13 @@ use sqlbuild_analysis::planner_fixtures::main::{ collection_mismatch::collection_mismatch, fixture_line::fixture_line, }; -use crate::bindings::_helpers::planner_strategy::constants::MODEL_REFERENCE_KIND; -use crate::bindings::_helpers::planner_strategy::fixture_completion::complete_fixtures; -use crate::bindings::_helpers::planner_strategy::fixture_sql::{ - complete_empty_fixture_sql, complete_typed_null_columns, -}; -use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::planner_strategy::types::FixtureKey; -use crate::bindings::_helpers::sql_test_glue::constants::FIXTURE_PREFIXES; +use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; +use crate::planner_strategy::main::complete_empty_fixture_sql::complete_empty_fixture_sql; +use crate::planner_strategy::main::complete_fixtures::complete_fixtures; +use crate::planner_strategy::main::complete_typed_null_columns::complete_typed_null_columns; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::planner_strategy::types::FixtureKey; +use crate::sql_test_glue::constants::FIXTURE_PREFIXES; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -24,7 +23,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::types::RecordId; /// Report every statically provable fixture defect or return the completed fixture SQL groups. -pub(super) fn validated_fixtures<'py>( +pub(crate) fn validated_fixtures<'py>( test: &Bound<'py, PyAny>, request: (&Bound<'py, PyAny>, [Bound<'py, PyDict>; 4]), inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -181,7 +180,7 @@ pub(super) fn validated_fixtures<'py>( } /// Return the test file and line that defines one fixture or expected-output CTE. -pub(super) fn fixture_location( +pub(crate) fn fixture_location( test: &Bound<'_, PyAny>, resource_type: &Bound<'_, PyAny>, name: &str, diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/mod.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/mod.rs new file mode 100644 index 0000000000..5dd33aa2e4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod cursor_window; +pub(crate) mod fixture_validation; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/mod.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/mod.rs new file mode 100644 index 0000000000..ea1ae2f6d1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod inputs; +pub(crate) mod plan; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/mod.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/mod.rs new file mode 100644 index 0000000000..62587e8392 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod plan_assembly; +pub(crate) mod plan_driver; +pub(crate) mod plan_entries; +pub(crate) mod plan_errors; +pub(crate) mod plan_fixtures; +pub(crate) mod plan_rendering; +pub(crate) mod plan_rows; +pub(crate) mod plan_values; +pub(crate) mod planning; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_assembly.rs similarity index 74% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_assembly.rs index df3e85d5e2..554d9c1d1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_assembly.rs @@ -3,11 +3,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; -use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::sql_test_glue::models::{SqlTestAssembly, SqlTestPlanOptions}; -use crate::bindings::_helpers::sql_test_glue::plan_driver::planned as planned_tests; +use crate::compiled_views::main::compiled_models::name_object::name_object; +use crate::planner_strategy::main::planning_context::planning_context; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::sql_test_glue::_helpers::plan::plan_driver::planned as planned_tests; +use crate::sql_test_glue::models::{SqlTestAssembly, SqlTestPlanOptions}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -15,7 +15,7 @@ use sqlbuild_python_base::records::types::RecordId; #[pyfunction] #[pyo3(signature = (**kwargs))] -pub(super) fn plan_sql_tests<'py>( +pub(crate) fn plan_sql_tests<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -52,21 +52,24 @@ pub(crate) fn assemble_sql_test_plans<'py>( ) -> PyResult> { let py = project.py(); let mut context: Option> = None; - let (planned, diagnostics) = if assembly.sql_analysis_enabled.is_truthy()? - && assembly.validate_fixtures - { - let given = assembly.fixture_planning_context.clone(); - let shared = if given.is_none() { - Bound::new(py, planning_context(project, None)?)? + let (planned, diagnostics) = + if assembly.sql_analysis_enabled.is_truthy()? && assembly.validate_fixtures { + let given = assembly.fixture_planning_context.clone(); + let shared = if given.is_none() { + Bound::new(py, planning_context(project, None)?)? + } else { + given.cast_into::()? + }; + let validated = crate::sql_test_glue::_helpers::plan::plan_fixtures::validate( + tests, + (project, adapter), + shared.get(), + )?; + context = Some(shared); + validated } else { - given.cast_into::()? + (tests.to_vec(), Default::default()) }; - let validated = super::plan_fixtures::validate(tests, (project, adapter), shared.get())?; - context = Some(shared); - validated - } else { - (tests.to_vec(), Default::default()) - }; let plannable: Vec<_> = planned .into_iter() .enumerate() @@ -92,7 +95,7 @@ pub(crate) fn assemble_sql_test_plans<'py>( let models = PyDict::new(py); for model in native_field(project, "models")?.try_iter()? { let model = model?; - models.set_item(model_reads::name_object(&model)?, model)?; + models.set_item(name_object(&model)?, model)?; } let mut results: Vec> = Vec::new(); for (index, test) in tests.iter().enumerate() { @@ -113,11 +116,13 @@ pub(crate) fn assemble_sql_test_plans<'py>( )); }; let mut errors: Vec = native_field(&plan, "error_messages")?.extract()?; - errors.extend(super::plan_entries::expected_diagnostics( - test, - &plan, - context.as_ref().map(Bound::get), - )?); + errors.extend( + crate::sql_test_glue::_helpers::plan::plan_entries::expected_diagnostics( + test, + &plan, + context.as_ref().map(Bound::get), + )?, + ); if !errors.is_empty() { results.push(new_record_with( py, @@ -135,7 +140,7 @@ pub(crate) fn assemble_sql_test_plans<'py>( &[ ( "entry", - super::plan_entries::entry( + crate::sql_test_glue::_helpers::plan::plan_entries::entry( test, &plan, (project, &models), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs similarity index 56% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs index 7cf82dd6ba..f935a7abb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs @@ -1,18 +1,17 @@ //! Host SQL-test request preparation, native planning, and Python result conversion. +use crate::sql_test_glue::main::planned_rows::planned_rows; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use sqlbuild_host::compile_context::main::add_compile_timing::add_compile_timing; +use pyo3::types::{PyDict, PyTuple}; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_strategy::fixture_context::count_chain_resolution; -use crate::bindings::_helpers::sql_test_glue::models::SqlTestPlanOptions; -use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; -use crate::bindings::_helpers::sql_test_glue::planning::{ - ChainRequest, PlanRow, PlanningRequest, plan_compiled_sql_tests, - resolve_compiled_sql_test_chains, +use crate::planner_phases::main::object::object; +use crate::planner_strategy::main::count_chain_resolution::count_chain_resolution; +use crate::sql_test_glue::_helpers::plan::plan_values::tuple; +use crate::sql_test_glue::_helpers::plan::planning::{ + ChainRequest, resolve_compiled_sql_test_chains, }; -use sqlbuild_python_base::boundary::main::imported::imported; +use crate::sql_test_glue::models::PlanRow; +use crate::sql_test_glue::models::SqlTestPlanOptions; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -22,7 +21,7 @@ use sqlbuild_python_base::records::types::RecordId; impl PlanRow { /// Rendered SQL for an artifact, or the planning error shared by native and Python consumers. - pub(crate) fn rendered_sql(&self, py: Python<'_>) -> PyResult<&str> { + pub fn rendered_sql(&self, py: Python<'_>) -> PyResult<&str> { let Some(sql) = self.plan.sql.as_deref() else { return Err(PyErr::from_value( error_class(py, ErrorId::NativeSqlTestPlanning)? @@ -33,7 +32,7 @@ impl PlanRow { } } -pub(super) fn chains<'py>( +pub(crate) fn chains<'py>( project: &Bound<'py, PyAny>, tests: &Bound<'py, PyAny>, ) -> PyResult> { @@ -55,7 +54,11 @@ pub(super) fn chains<'py>( let native = match native { Ok(value) => value, Err(error) if error.is_instance_of::(py) => { - return Err(super::plan_errors::planning_error(py, error, tests)?); + return Err( + crate::sql_test_glue::_helpers::plan::plan_errors::planning_error( + py, error, tests, + )?, + ); } Err(error) => return Err(error), }; @@ -75,81 +78,6 @@ pub(crate) fn resolve_sql_test_model_chains<'py>( chains(project, tests) } -/// Native plans of one SQL-test batch, with adapter rendering and timing accounted for. -pub(crate) fn planned_rows<'py>( - (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - tests: &Bound<'py, PyAny>, - options: &SqlTestPlanOptions<'py>, -) -> PyResult> { - let py = project.py(); - let artifact = options.artifact; - if !tests.is_truthy()? { - return Ok(Vec::new()); - } - let start: u64 = imported(py, "time", "perf_counter_ns")? - .call0()? - .extract()?; - let model_sql = super::plan_rendering::models(project, adapter)?; - let functions = super::plan_rendering::functions(project, adapter)?; - let overrides = super::plan_rendering::overrides(project, tests, adapter)?; - let request = PlanningRequest { - models: native_field(project, "models")?.extract()?, - rendered_model_sql: model_sql.extract()?, - functions: functions.extract()?, - tests: tests.extract()?, - rendered_test_overrides: overrides.extract()?, - sql_analysis_enabled: options.sql_analysis_enabled.extract()?, - sql_analysis_dialect: adapter.call_method0("sql_analysis_dialect")?.extract()?, - set_difference_operator: adapter - .call_method0("render_set_difference_operator")? - .extract()?, - requires_derived_table_aliases: adapter - .call_method0("requires_derived_table_aliases")? - .extract()?, - lexical_syntax: native_field( - &native_field(project, "sql_lexical_syntax")?, - "native_mapping", - )? - .extract()?, - render_sql: artifact || options.render_sql, - include_plan: !artifact && options.include_plan, - max_unit_test_real_models: native_field( - &native_field(project, "settings")?, - "unit_test_max_real_models", - )? - .extract()?, - }; - let (rows, planning_ns, rendering_ns) = match plan_compiled_sql_tests(py, request) { - Ok(value) => value, - Err(error) if error.is_instance_of::(py) => { - return Err(super::plan_errors::planning_error(py, error, tests)?); - } - Err(error) => return Err(error), - }; - let end: u64 = imported(py, "time", "perf_counter_ns")? - .call0()? - .extract()?; - for (phase, elapsed) in [ - ( - "test_planning_ms", - planning_ns - + end - .saturating_sub(start) - .saturating_sub(planning_ns) - .saturating_sub(rendering_ns), - ), - ("comparison_render_ms", rendering_ns), - ] { - add_compile_timing(phase, elapsed); - } - if artifact { - for row in &rows { - row.rendered_sql(py)?; - } - } - Ok(rows) -} - /// Plans of one SQL-test batch, or their compiled artifacts, for Python consumers. pub(crate) fn planned<'py>( (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -160,7 +88,7 @@ pub(crate) fn planned<'py>( let rows = planned_rows((project, adapter), tests, options)?; let mut plans: Vec> = Vec::new(); for row in rows { - let plan = super::plan_rows::plan(py, row)?; + let plan = crate::sql_test_glue::_helpers::plan::plan_rows::plan(py, row)?; if options.artifact { let sql = native_field(&plan, "sql")?; plans.push(new_record_with( @@ -181,7 +109,7 @@ pub(crate) fn planned<'py>( #[pyfunction] #[pyo3(signature = (**kwargs))] -pub(super) fn plan_sql_tests_natively<'py>( +pub(crate) fn plan_sql_tests_natively<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -250,7 +178,7 @@ pub(crate) fn sql_test_plan_error_messages<'py>( let warning = warning?; let message: String = native_field(&warning, "message")?.extract()?; if native_field(&native_field(&warning, "severity")?, "value")?.extract::()? - == super::constants::ERROR_SEVERITY + == crate::sql_test_glue::constants::ERROR_SEVERITY && seen.insert(message.clone()) { messages.push(message); @@ -258,19 +186,3 @@ pub(crate) fn sql_test_plan_error_messages<'py>( } PyTuple::new(warnings.py(), messages) } - -/// Return model closures resolved natively in one project batch, keyed by SQL test key. -#[pyfunction] -#[pyo3(signature = (*, project, tests))] -pub(crate) fn sql_test_model_chain_names_by_key<'py>( - project: &Bound<'py, PyAny>, - tests: &Bound<'py, PyAny>, -) -> PyResult> { - let py = project.py(); - let chains = resolve_sql_test_model_chains(project, tests)?; - let by_key = PyDict::new(py); - for (test, chain) in tests.try_iter()?.zip(chains.try_iter()?) { - by_key.set_item(native_field(&test?, "key")?, chain?)?; - } - Ok(by_key) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs similarity index 92% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs index 01f3019e20..8e2028dd65 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs @@ -6,10 +6,10 @@ use sqlbuild_model_config::execution_limits::constants::TIMEOUT_KEY as TIMEOUT_H use sqlbuild_model_config::execution_limits::main::parse_execution_limit::parse_execution_limit; use sqlbuild_model_config::execution_limits::models::ExecutionLimit; -use crate::bindings::_helpers::planner_strategy::constants::MODEL_REFERENCE_KIND; -use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::sql_test_glue::constants::{TABLE_FN, UDF}; -use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; +use crate::planner_strategy::constants::MODEL_REFERENCE_KIND; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::sql_test_glue::_helpers::plan::plan_values::model_payload; +use crate::sql_test_glue::constants::{TABLE_FN, UDF}; use sqlbuild_analysis::compiler::main::missing_expected_columns::missing_expected_columns; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; @@ -91,7 +91,7 @@ fn function_deps<'py>( Ok(PyTuple::new(py, deps.keys().iter())?.into_any()) } -pub(super) fn entry<'py>( +pub(crate) fn entry<'py>( test: &Bound<'py, PyAny>, plan: &Bound<'py, PyAny>, inputs: (&Bound<'py, PyAny>, &Bound<'py, PyDict>), @@ -154,7 +154,7 @@ pub(super) fn entry<'py>( new_record(py, RecordId::SqlTestPlanEntry, (), Some(&fields)) } -pub(super) fn expected_diagnostics( +pub(crate) fn expected_diagnostics( test: &Bound<'_, PyAny>, plan: &Bound<'_, PyAny>, context: Option<&RelationFixturePlanningContext>, @@ -186,11 +186,12 @@ pub(super) fn expected_diagnostics( else { continue; }; - let location = super::fixture_validation::fixture_location( - test, - &native_field(&resource_types, "SQL_TEST")?, - &name.extract::()?, - )?; + let location = + crate::sql_test_glue::_helpers::inputs::fixture_validation::fixture_location( + test, + &native_field(&resource_types, "SQL_TEST")?, + &name.extract::()?, + )?; result.push(format!( "SQL test '{}': {}: {}", native_field(test, "name")?.extract::()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_errors.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_errors.rs index ceda5bea27..1d5a35b824 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_errors.rs @@ -1,6 +1,6 @@ //! Project native planning failures into existing located diagnostic exceptions. -use crate::bindings::_helpers::sql_test_glue::plan_values::string; +use crate::sql_test_glue::_helpers::plan::plan_values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::sql_test_attachment::main::sql_test_cte_location::sql_test_cte_location; @@ -35,7 +35,7 @@ fn cte_call_location<'py>( new_record(py, RecordId::SourceLocation, (), Some(&arguments)) } -pub(super) fn planning_error( +pub(crate) fn planning_error( py: Python<'_>, error: PyErr, tests: &Bound<'_, PyAny>, @@ -73,7 +73,10 @@ pub(super) fn planning_error( kwargs.set_item("location", cte_call_location(py, &test, (&cte, &call))?)?; ErrorId::SqlTestReference } else { - kwargs.set_item("code", super::constants::SQL_TEST_HELPER_REFERENCE_CODE)?; + kwargs.set_item( + "code", + crate::sql_test_glue::constants::SQL_TEST_HELPER_REFERENCE_CODE, + )?; ErrorId::CompileInput }; return Ok(PyErr::from_value( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_fixtures.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_fixtures.rs index 4e5953ee77..52631017fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_fixtures.rs @@ -3,16 +3,16 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::sql_test_glue::fixture_validation::validated_fixtures; -use crate::bindings::_helpers::sql_test_glue::plan_driver::resolve_sql_test_model_chains; -use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; +use crate::planner_strategy::models::RelationFixturePlanningContext; +use crate::sql_test_glue::_helpers::inputs::fixture_validation::validated_fixtures; +use crate::sql_test_glue::_helpers::plan::plan_driver::resolve_sql_test_model_chains; +use crate::sql_test_glue::_helpers::plan::plan_values::model_payload; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_front::sql_test_glue::main::prefixed_ctes::prefixed_ctes; -pub(super) fn validated_test<'py>( +pub(crate) fn validated_test<'py>( test: &Bound<'py, PyAny>, chain: &Bound<'py, PyAny>, inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -76,11 +76,11 @@ pub(super) fn validated_test<'py>( replaced_keywords(test, &kwargs).map(Ok) } -pub(super) fn validate<'py>( +pub(crate) fn validate<'py>( tests: &[Bound<'py, PyAny>], inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), context: &RelationFixturePlanningContext, -) -> PyResult> { +) -> PyResult> { let py = inputs.0.py(); let mut model_tests: Vec> = Vec::new(); let mut indexes: Vec = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs index 4aad8b40c1..5201e859d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs @@ -1,26 +1,27 @@ //! Adapter rendering and declared cursor-window boundaries for native test planning. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::sql_test_glue::plan_values::{model_payload, string}; +use crate::compiled_views::main::compiled_models::name_object::name_object; +use crate::compiled_views::main::compiled_models::query_sql::query_sql; +use crate::sql_test_glue::_helpers::plan::plan_values::{model_payload, string}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; -use crate::bindings::_helpers::sql_test_glue::constants::SENTINEL; -use crate::bindings::_helpers::sql_test_glue::cursor_window::{ +use crate::sql_test_glue::_helpers::inputs::cursor_window::{ declared_window_cursor_models, declares_cursor_window, render_test_cursor_intrinsics, }; +use crate::sql_test_glue::constants::SENTINEL; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; -pub(super) fn mentions(sql: &str) -> bool { +pub(crate) fn mentions(sql: &str) -> bool { ["__cursor_start", "__cursor_end"] .iter() .any(|name| sql.contains(name)) } -pub(super) fn models<'py>( +pub(crate) fn models<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, ) -> PyResult> { @@ -28,7 +29,7 @@ pub(super) fn models<'py>( let result = PyDict::new(py); for (index, model) in native_field(project, "models")?.try_iter()?.enumerate() { let model = model?; - let sql = model_reads::query_sql(&model)?; + let sql = query_sql(&model)?; if mentions(&sql.extract::()?) { result.set_item( index, @@ -39,7 +40,7 @@ pub(super) fn models<'py>( Ok(result) } -pub(super) fn overrides<'py>( +pub(crate) fn overrides<'py>( project: &Bound<'py, PyAny>, tests: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, @@ -48,7 +49,7 @@ pub(super) fn overrides<'py>( let models = PyDict::new(py); for model in native_field(project, "models")?.try_iter()? { let model = model?; - models.set_item(model_reads::name_object(&model)?, model)?; + models.set_item(name_object(&model)?, model)?; } let tests: Vec> = tests.extract()?; let mut indexes: Vec = Vec::new(); @@ -62,8 +63,11 @@ pub(super) fn overrides<'py>( let chains = if indexes.is_empty() { Vec::new() } else { - super::plan_driver::chains(project, &PyTuple::new(py, windowed)?.into_any())? - .extract::>>()? + crate::sql_test_glue::_helpers::plan::plan_driver::chains( + project, + &PyTuple::new(py, windowed)?.into_any(), + )? + .extract::>>()? }; if indexes.len() != chains.len() { return Err(pyo3::exceptions::PyValueError::new_err( @@ -116,12 +120,12 @@ pub(super) fn overrides<'py>( } } for model in declared_window_cursor_models(test, chain_models)? { - let name = model_reads::name_object(&model)?; + let name = name_object(&model)?; if !rendered.contains(&name)? { rendered.set_item( name, render_test_cursor_intrinsics( - model_reads::query_sql(&model)?, + query_sql(&model)?, &model, adapter, Some(test), @@ -135,7 +139,7 @@ pub(super) fn overrides<'py>( Ok(result) } -pub(super) fn functions<'py>( +pub(crate) fn functions<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rows.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rows.rs index 2e521f1627..bc8ac59e81 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rows.rs @@ -1,6 +1,6 @@ //! Materialize native plan rows as the existing executor-facing dataclasses. -use crate::bindings::_helpers::sql_test_glue::planning::PlanRow; +use crate::sql_test_glue::models::PlanRow; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyTuple; @@ -64,7 +64,7 @@ fn assertion_step<'py>(py: Python<'py>, step: SqlTestAssertionStep) -> PyResult< ) } -pub(super) fn plan(py: Python<'_>, row: PlanRow) -> PyResult> { +pub(crate) fn plan(py: Python<'_>, row: PlanRow) -> PyResult> { let PlanRow { plan: SqlTestPlan { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_values.rs similarity index 79% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_values.rs index fad31f5a37..0b09a2b44b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_values.rs @@ -8,17 +8,17 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -pub(super) fn string<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { +pub(crate) fn string<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() } -pub(super) fn model_payload(test: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn model_payload(test: &Bound<'_, PyAny>) -> PyResult { native_field(test, "payload")?.is_instance(&record_class( test.py(), RecordId::CompiledModelSqlTestPayload, )?) } -pub(super) fn tuple<'py>(value: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn tuple<'py>(value: &Bound<'py, PyAny>) -> PyResult> { imported(value.py(), "builtins", "tuple")?.call1((value,)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/planning.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs rename to crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/planning.rs index 45a0cfcced..f8c71ce1ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs +++ b/crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/planning.rs @@ -1,5 +1,6 @@ //! Plan compiled SQL tests from the compiled project's own objects, without a JSON request. +use crate::sql_test_glue::models::PlanRow; use sqlbuild_python_base::positional_record; use std::collections::{BTreeMap, HashMap}; @@ -12,7 +13,8 @@ use sqlbuild_analysis::compiler::models::{ SqlTestPlanFunction, SqlTestPlanModel, SqlTestPlanPayload, SqlTestPlanTest, }; -use crate::bindings::_helpers::compiled_views::model_reads; +use crate::compiled_views::main::compiled_models::name_object::name_object; +use crate::compiled_views::main::compiled_models::query_sql::query_sql; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; @@ -21,12 +23,6 @@ use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; const MODEL_RESOURCE_TYPE: &str = "model"; const PLANNING_WORKERS: usize = 4; -/// A native SQL-test plan with its distinct error messages. -pub(crate) struct PlanRow { - pub(crate) plan: SqlTestPlan, - pub(crate) error_messages: Vec, -} - positional_record! { extract; /// An SQL function's name with its rendered UDF and table-function call affixes. @@ -76,7 +72,7 @@ pub(crate) fn resolve_compiled_sql_test_chains( .iter() .map(|model| { Ok(SqlTestChainModel { - name: model_reads::name_object(model)?.extract()?, + name: name_object(model)?.extract()?, model_dependencies: model_dependencies(model)?, }) }) @@ -142,10 +138,10 @@ fn plan_model( rendered_sql: Option, ) -> PyResult { Ok(SqlTestPlanModel { - name: model_reads::name_object(model)?.extract()?, + name: name_object(model)?.extract()?, query_sql: match rendered_sql { Some(sql) => sql, - None => model_reads::query_sql(model)?.extract()?, + None => query_sql(model)?.extract()?, }, model_dependencies: model_dependencies(model)?, }) diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/constants.rs b/crates/sqlbuild-python-project/src/sql_test_glue/constants.rs new file mode 100644 index 0000000000..2983e7a114 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/constants.rs @@ -0,0 +1,43 @@ +//! Public constants of sql test glue. + +pub(crate) const UDF: &str = "udf"; + +pub(crate) const TABLE_FN: &str = "table_fn"; + +pub(crate) const ERROR_SEVERITY: &str = "error"; + +pub(crate) const FIXTURE_PREFIXES: [(&str, &str); 4] = [ + ("model", "__ref__"), + ("source", "__source__"), + ("seed", "__seed__"), + ("sql_test", "__expected__"), +]; + +pub(crate) const CURSOR_ALGEBRA_EXCEPTIONS: &str = "sqlbuild.cursor_algebra.exceptions"; + +pub(crate) const INTEGER_CURSOR_TYPE: &str = "integer"; + +pub(crate) const TIMESTAMP_CURSOR_TYPE: &str = "timestamp"; + +pub const DEFAULT_TIMESTAMP_START: &str = "1900-01-01 00:00:00"; + +pub const DEFAULT_TIMESTAMP_END: &str = "2999-12-31 00:00:00"; + +pub const DEFAULT_INTEGER_START: &str = "-1000000000000000"; + +pub const DEFAULT_INTEGER_END: &str = "1000000000000000"; + +pub(crate) const CURSOR_START_KEY: &str = "cursor_start"; + +pub(crate) const CURSOR_END_KEY: &str = "cursor_end"; + +pub(crate) const SENTINEL: &str = "__SQLBUILD_CALL_SUFFIX__"; + +/// The diagnostic code of a SQL test that references a helper it may not read. +pub const SQL_TEST_HELPER_REFERENCE_CODE: &str = "P013"; + +/// The scan store namespace of expanded SQL-test CTEs. +pub const SQL_TEST_CTE_SCAN_ALGORITHM: &str = "expanded-sql-test-ctes-v1"; + +/// The scan store namespace of the models SQL tests expect. +pub const SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM: &str = "sql-test-expected-models-v1"; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/assemble_sql_test_plans.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/assemble_sql_test_plans.rs new file mode 100644 index 0000000000..3a196ffab5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/assemble_sql_test_plans.rs @@ -0,0 +1,16 @@ +//! Plan results of SQL tests, with fixture and expected-output diagnostics in place of failing plans. + +use crate::sql_test_glue::_helpers::plan::plan_assembly; +use crate::sql_test_glue::models::SqlTestAssembly; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyTuple; + +/// Plan results of SQL tests, with fixture and expected-output diagnostics in place of failing plans. +#[inline] +pub fn assemble_sql_test_plans<'py>( + tests: &[Bound<'py, PyAny>], + (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + assembly: &SqlTestAssembly<'py>, +) -> PyResult> { + plan_assembly::assemble_sql_test_plans(tests, (project, adapter), assembly) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/mod.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/mod.rs new file mode 100644 index 0000000000..5d3bad2924 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/mod.rs @@ -0,0 +1,8 @@ +pub mod assemble_sql_test_plans; +pub mod plan_compiled_sql_test_artifacts_function; +pub mod plan_sql_tests_function; +pub mod plan_sql_tests_natively_function; +pub mod planned_rows; +pub mod sql_test_model_chain_names_by_key; +pub mod sql_test_model_chain_names_by_key_function; +pub mod sql_test_plan_error_messages_function; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_compiled_sql_test_artifacts_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_compiled_sql_test_artifacts_function.rs new file mode 100644 index 0000000000..cb78440ec2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_compiled_sql_test_artifacts_function.rs @@ -0,0 +1,10 @@ +//! `plan_compiled_sql_test_artifacts` as a function of `module`. + +use crate::sql_test_glue::_helpers::plan::plan_driver::plan_compiled_sql_test_artifacts; + +/// `plan_compiled_sql_test_artifacts` as a function of `module`. +pub fn plan_compiled_sql_test_artifacts_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(plan_compiled_sql_test_artifacts, module) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_function.rs new file mode 100644 index 0000000000..123b1d6d63 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_function.rs @@ -0,0 +1,10 @@ +//! `plan_sql_tests` as a function of `module`. + +use crate::sql_test_glue::_helpers::plan::plan_assembly::plan_sql_tests; + +/// `plan_sql_tests` as a function of `module`. +pub fn plan_sql_tests_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(plan_sql_tests, module) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_natively_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_natively_function.rs new file mode 100644 index 0000000000..445d8bf4ec --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/plan_sql_tests_natively_function.rs @@ -0,0 +1,10 @@ +//! `plan_sql_tests_natively` as a function of `module`. + +use crate::sql_test_glue::_helpers::plan::plan_driver::plan_sql_tests_natively; + +/// `plan_sql_tests_natively` as a function of `module`. +pub fn plan_sql_tests_natively_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(plan_sql_tests_natively, module) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs new file mode 100644 index 0000000000..a5509174c7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs @@ -0,0 +1,90 @@ +//! Native plans of one SQL-test batch, with adapter rendering and timing accounted for. + +use crate::sql_test_glue::_helpers::plan::planning::{PlanningRequest, plan_compiled_sql_tests}; +use crate::sql_test_glue::models::PlanRow; +use crate::sql_test_glue::models::SqlTestPlanOptions; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_host::compile_context::main::add_compile_timing::add_compile_timing; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Native plans of one SQL-test batch, with adapter rendering and timing accounted for. +pub fn planned_rows<'py>( + (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + tests: &Bound<'py, PyAny>, + options: &SqlTestPlanOptions<'py>, +) -> PyResult> { + let py = project.py(); + let artifact = options.artifact; + if !tests.is_truthy()? { + return Ok(Vec::new()); + } + let start: u64 = imported(py, "time", "perf_counter_ns")? + .call0()? + .extract()?; + let model_sql = crate::sql_test_glue::_helpers::plan::plan_rendering::models(project, adapter)?; + let functions = + crate::sql_test_glue::_helpers::plan::plan_rendering::functions(project, adapter)?; + let overrides = + crate::sql_test_glue::_helpers::plan::plan_rendering::overrides(project, tests, adapter)?; + let request = PlanningRequest { + models: native_field(project, "models")?.extract()?, + rendered_model_sql: model_sql.extract()?, + functions: functions.extract()?, + tests: tests.extract()?, + rendered_test_overrides: overrides.extract()?, + sql_analysis_enabled: options.sql_analysis_enabled.extract()?, + sql_analysis_dialect: adapter.call_method0("sql_analysis_dialect")?.extract()?, + set_difference_operator: adapter + .call_method0("render_set_difference_operator")? + .extract()?, + requires_derived_table_aliases: adapter + .call_method0("requires_derived_table_aliases")? + .extract()?, + lexical_syntax: native_field( + &native_field(project, "sql_lexical_syntax")?, + "native_mapping", + )? + .extract()?, + render_sql: artifact || options.render_sql, + include_plan: !artifact && options.include_plan, + max_unit_test_real_models: native_field( + &native_field(project, "settings")?, + "unit_test_max_real_models", + )? + .extract()?, + }; + let (rows, planning_ns, rendering_ns) = match plan_compiled_sql_tests(py, request) { + Ok(value) => value, + Err(error) if error.is_instance_of::(py) => { + return Err( + crate::sql_test_glue::_helpers::plan::plan_errors::planning_error( + py, error, tests, + )?, + ); + } + Err(error) => return Err(error), + }; + let end: u64 = imported(py, "time", "perf_counter_ns")? + .call0()? + .extract()?; + for (phase, elapsed) in [ + ( + "test_planning_ms", + planning_ns + + end + .saturating_sub(start) + .saturating_sub(planning_ns) + .saturating_sub(rendering_ns), + ), + ("comparison_render_ms", rendering_ns), + ] { + add_compile_timing(phase, elapsed); + } + if artifact { + for row in &rows { + row.rendered_sql(py)?; + } + } + Ok(rows) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key.rs new file mode 100644 index 0000000000..d961845f01 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key.rs @@ -0,0 +1,22 @@ +//! Return model closures resolved natively in one project batch, keyed by SQL test key. + +use crate::sql_test_glue::_helpers::plan::plan_driver::resolve_sql_test_model_chains; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Return model closures resolved natively in one project batch, keyed by SQL test key. +#[pyfunction] +#[pyo3(signature = (*, project, tests))] +pub fn sql_test_model_chain_names_by_key<'py>( + project: &Bound<'py, PyAny>, + tests: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let chains = resolve_sql_test_model_chains(project, tests)?; + let by_key = PyDict::new(py); + for (test, chain) in tests.try_iter()?.zip(chains.try_iter()?) { + by_key.set_item(native_field(&test?, "key")?, chain?)?; + } + Ok(by_key) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key_function.rs new file mode 100644 index 0000000000..19b88aa2de --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_model_chain_names_by_key_function.rs @@ -0,0 +1,10 @@ +//! `sql_test_model_chain_names_by_key` as a function of `module`. + +use crate::sql_test_glue::main::sql_test_model_chain_names_by_key::sql_test_model_chain_names_by_key; + +/// `sql_test_model_chain_names_by_key` as a function of `module`. +pub fn sql_test_model_chain_names_by_key_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(sql_test_model_chain_names_by_key, module) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_plan_error_messages_function.rs b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_plan_error_messages_function.rs new file mode 100644 index 0000000000..66a3326ea3 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/main/sql_test_plan_error_messages_function.rs @@ -0,0 +1,10 @@ +//! `sql_test_plan_error_messages` as a function of `module`. + +use crate::sql_test_glue::_helpers::plan::plan_driver::sql_test_plan_error_messages; + +/// `sql_test_plan_error_messages` as a function of `module`. +pub fn sql_test_plan_error_messages_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(sql_test_plan_error_messages, module) +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/mod.rs b/crates/sqlbuild-python-project/src/sql_test_glue/mod.rs new file mode 100644 index 0000000000..7938daf0af --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/mod.rs @@ -0,0 +1,7 @@ +//! SQL test planning: fixtures, entries, rows, rendering, errors and cursor windows. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/models.rs b/crates/sqlbuild-python-project/src/sql_test_glue/models.rs new file mode 100644 index 0000000000..2bfa2fd81c --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/models.rs @@ -0,0 +1,28 @@ +//! Public records of sql test glue. + +use pyo3::prelude::{Bound, PyAny}; +use sqlbuild_analysis::compiler::models::SqlTestPlan; + +/// How one batch of SQL tests is planned. +#[derive(Debug)] +pub struct SqlTestPlanOptions<'py> { + pub sql_analysis_enabled: Bound<'py, PyAny>, + pub render_sql: bool, + pub include_plan: bool, + pub artifact: bool, +} + +/// How assembling SQL-test plan results validates fixtures. +#[derive(Debug)] +pub struct SqlTestAssembly<'py> { + pub sql_analysis_enabled: Bound<'py, PyAny>, + pub validate_fixtures: bool, + pub fixture_planning_context: Bound<'py, PyAny>, +} + +/// A native SQL-test plan with its distinct error messages. +#[derive(Debug)] +pub struct PlanRow { + pub plan: SqlTestPlan, + pub error_messages: Vec, +} diff --git a/crates/sqlbuild-python-project/src/sql_test_glue/types.rs b/crates/sqlbuild-python-project/src/sql_test_glue/types.rs new file mode 100644 index 0000000000..6660825598 --- /dev/null +++ b/crates/sqlbuild-python-project/src/sql_test_glue/types.rs @@ -0,0 +1,6 @@ +//! Public traits and type aliases of sql test glue. + +use pyo3::prelude::{Bound, PyAny}; +use std::collections::BTreeMap; + +pub(crate) type ValidatedTests<'py> = (Vec>, BTreeMap>); diff --git a/crates/sqlbuild-python-project/src/type_system/_helpers/mod.rs b/crates/sqlbuild-python-project/src/type_system/_helpers/mod.rs new file mode 100644 index 0000000000..c27ad92817 --- /dev/null +++ b/crates/sqlbuild-python-project/src/type_system/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod nullability; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/nullability.rs b/crates/sqlbuild-python-project/src/type_system/_helpers/nullability.rs similarity index 75% rename from crates/sqlbuild-python/src/bindings/_helpers/type_system/nullability.rs rename to crates/sqlbuild-python-project/src/type_system/_helpers/nullability.rs index 978b62389c..4f80f8b17e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/nullability.rs +++ b/crates/sqlbuild-python-project/src/type_system/_helpers/nullability.rs @@ -1,8 +1,8 @@ //! Adapter nullability rules over `InferredNullability` arguments, decided by the native rules. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyString; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::nullability_rules::main::conditional_result_nullability::conditional_result_nullability; use sqlbuild_analysis::nullability_rules::main::safe_cast_nullability::safe_cast_nullability; @@ -34,7 +34,9 @@ fn argument_values(args: &Bound<'_, PyAny>, count: usize) -> PyResult(args: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn infer_first_arg_nullability<'py>( + args: &Bound<'py, PyAny>, +) -> PyResult> { if !args.is_truthy()? { return enum_member(args.py(), RecordId::InferredNullability, UNKNOWN_MEMBER); } @@ -43,7 +45,7 @@ fn infer_first_arg_nullability<'py>(args: &Bound<'py, PyAny>) -> PyResult( +pub(crate) fn infer_conditional_result_nullability<'py>( args: &Bound<'py, PyAny>, ) -> PyResult> { let values: Vec = argument_values(args, args.len()?)?; @@ -53,7 +55,9 @@ fn infer_conditional_result_nullability<'py>( /// SAFE_CAST/TRY_CAST nullability without assuming the conversion succeeds. #[pyfunction] -fn infer_safe_cast_nullability<'py>(args: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn infer_safe_cast_nullability<'py>( + args: &Bound<'py, PyAny>, +) -> PyResult> { let values: Vec = if args.is_truthy()? { argument_values(args, 1)? } else { @@ -62,13 +66,3 @@ fn infer_safe_cast_nullability<'py>(args: &Bound<'py, PyAny>) -> PyResult = values.iter().map(String::as_str).collect(); member(args, safe_cast_nullability(&values)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(infer_first_arg_nullability, module)?)?; - module.add_function(wrap_pyfunction!( - infer_conditional_result_nullability, - module - )?)?; - module.add_function(wrap_pyfunction!(infer_safe_cast_nullability, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-project/src/type_system/main/mod.rs b/crates/sqlbuild-python-project/src/type_system/main/mod.rs new file mode 100644 index 0000000000..20e7f1fc56 --- /dev/null +++ b/crates/sqlbuild-python-project/src/type_system/main/mod.rs @@ -0,0 +1 @@ +pub mod register_nullability; diff --git a/crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs b/crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs new file mode 100644 index 0000000000..867b57758e --- /dev/null +++ b/crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs @@ -0,0 +1,18 @@ +//! Register the nullability functions and classes. + +use crate::type_system::_helpers::nullability::infer_conditional_result_nullability; +use crate::type_system::_helpers::nullability::infer_first_arg_nullability; +use crate::type_system::_helpers::nullability::infer_safe_cast_nullability; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the nullability functions and classes. +pub fn register_nullability(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(infer_first_arg_nullability, module)?)?; + module.add_function(wrap_pyfunction!( + infer_conditional_result_nullability, + module + )?)?; + module.add_function(wrap_pyfunction!(infer_safe_cast_nullability, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-project/src/type_system/mod.rs b/crates/sqlbuild-python-project/src/type_system/mod.rs new file mode 100644 index 0000000000..3b579f414a --- /dev/null +++ b/crates/sqlbuild-python-project/src/type_system/mod.rs @@ -0,0 +1,4 @@ +//! Nullability of SQL types. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 5296294002..8170d2c1ad 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -17,7 +17,6 @@ extension-module = ["pyo3/extension-module"] [dependencies] chrono.workspace = true -chrono-tz.workspace = true fensu-policy.workspace = true pyo3.workspace = true rayon.workspace = true @@ -25,7 +24,6 @@ rusqlite.workspace = true rustc-hash.workspace = true serde.workspace = true serde_json.workspace = true -sha2.workspace = true signal-hook.workspace = true sqlbuild-analysis.workspace = true sqlbuild-attachments.workspace = true @@ -42,6 +40,7 @@ sqlbuild-project.workspace = true sqlbuild-pysource.workspace = true sqlbuild-python-base.workspace = true sqlbuild-python-front.workspace = true +sqlbuild-python-project.workspace = true sqlbuild-recall.workspace = true sqlbuild-refactor.workspace = true sqlbuild-render.workspace = true @@ -50,7 +49,6 @@ sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlbuild-state.workspace = true sqlclean.workspace = true -tokio.workspace = true uuid.workspace = true [lints] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs index 4a2c0e4e79..e60427317e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs @@ -10,7 +10,7 @@ use sqlbuild_analysis::binding_locations::models::{ use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; use sqlbuild_python_base::positional_record; -use crate::bindings::models::Positional; +use sqlbuild_python_project::analysis::models::Positional; positional_record! { into; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/getters.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/getters.rs index 8e6a305bf8..445fe009fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/getters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/getters.rs @@ -3,9 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::analysis::binding_catalog::BindingCatalog; use sqlbuild_python_base::records::main::registry::class_of::class_of; use sqlbuild_python_base::records::models::GetterEntry; +use sqlbuild_python_project::analysis::models::BindingCatalog; /// The getter readers of this module's native pyclasses. pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs index 39fd6f893b..f896a0e29c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs @@ -1,14 +1,6 @@ -pub(crate) mod binding_catalog; pub(crate) mod binding_locations; -mod catalog; -pub(crate) mod conversions; pub(crate) mod getters; -pub(crate) mod normalization; -pub(crate) mod normalization_results; -mod positions; pub(crate) mod queries; -pub(crate) mod relation_names; mod required_columns; -pub(crate) mod schema_request; pub(crate) mod sql_test_comparisons; pub(crate) mod sql_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs index 249917466f..322bfa51a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs @@ -5,6 +5,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::analysis::main::register_binding_catalog::register_binding_catalog; +use sqlbuild_python_project::references::main::register_explicit_references::register_explicit_references; #[pyfunction] fn query_fingerprint(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { @@ -27,9 +29,9 @@ fn parse_warehouse_type(py: Python<'_>, spelling: &str, dialect: &str) -> PyResu } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - crate::bindings::_helpers::analysis::binding_catalog::register(module)?; + register_binding_catalog(module)?; crate::bindings::_helpers::analysis::required_columns::register(module)?; - crate::bindings::_helpers::references::explicit_references::register(module)?; + register_explicit_references(module)?; crate::bindings::_helpers::analysis::binding_locations::register(module)?; module.add_function(wrap_pyfunction!(query_fingerprint, module)?)?; module.add_function(wrap_pyfunction!(parse_warehouse_type, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/relation_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/relation_names.rs deleted file mode 100644 index a5aee1a374..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/relation_names.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! Relations named in SQL and temporary tables it creates, for hard-coded name checks. - -use pyo3::PyErr; -use pyo3::prelude::{PyResult, Python}; -use sqlbuild_analysis::relation_names::errors::RelationNamesFailure; -use sqlbuild_analysis::relation_names::main::extract_relation_names::extract_relation_names; -use sqlbuild_analysis::relation_names::models::{ExtractedRelations, RelationName}; -use sqlbuild_core::panics::main::native_failure::native_failure; - -use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; -use sqlbuild_python_base::types::CompilerDetach; - -const RELATION_NAMES_CONTEXT: &str = "native relation name extraction"; - -/// `(name, schema, database)`. -pub(crate) type RelationRow = (String, Option, Option); - -/// `(relations, temporary names)` named in `sql`, or None if unparseable. -pub(crate) fn extract_sql_relation_names( - py: Python<'_>, - sql: &str, - dialect: Option<&str>, -) -> PyResult, Vec)>> { - let extracted = py - .compiler_detach(|| Ok(extract_relation_names(sql, dialect))) - .map_err(compiler_error)? - .map_err(relation_names_error)?; - Ok(extracted.map( - |ExtractedRelations { - relations, - temporary, - }| { (relations.into_iter().map(relation_row).collect(), temporary) }, - )) -} - -fn relation_row(relation: RelationName) -> RelationRow { - (relation.name, relation.schema, relation.database) -} - -/// An unknown dialect as the wheel's `ValueError`; anything else as an internal failure. -fn relation_names_error(failure: RelationNamesFailure) -> PyErr { - match failure { - RelationNamesFailure::UnknownDialect(message) => compiler_error(message), - RelationNamesFailure::Internal(reason) => { - compiler_error(native_failure(RELATION_NAMES_CONTEXT, &reason)) - } - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs deleted file mode 100644 index e676df32f8..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! Fixture column inference computed once per fixture body and inference profile. - -use crate::bindings::_helpers::analysis_session::fixture_columns::infer_fixture_column_facts; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyclass, pymethods}; -use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use std::sync::Mutex; - -/// Reuse inference for identical fixture SQL under an equal, catalog-free profile. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct FixtureColumnInferences { - by_profile: Mutex, Py)>>, -} - -#[pymethods] -impl FixtureColumnInferences { - #[new] - pub(crate) fn new() -> Self { - Self { - by_profile: Mutex::new(Vec::new()), - } - } - - /// The inference for one fixture query, computing each distinct query once. - #[pyo3(signature = (*, query_sql, inference_profile))] - fn infer<'py>( - &self, - query_sql: &Bound<'py, PyAny>, - inference_profile: &Bound<'py, PyAny>, - ) -> PyResult> { - let py = query_sql.py(); - let compute = || { - infer_fixture_column_facts(Some(&keywords( - py, - &[ - ("query_sql", query_sql.clone()), - ("inference_profile", inference_profile.clone()), - ], - )?)) - }; - if !native_field(inference_profile, "binding_catalog")?.is_none() { - return compute(); - } - let inferred = self.profile_results(py, inference_profile)?; - if let Some(found) = inferred.get_item(query_sql)? { - return Ok(found); - } - let computed = compute()?; - inferred.set_item(query_sql, &computed)?; - Ok(computed) - } -} - -impl FixtureColumnInferences { - fn profile_results<'py>( - &self, - py: Python<'py>, - inference_profile: &Bound<'py, PyAny>, - ) -> PyResult> { - let known: Vec<(Py, Py)> = self - .by_profile - .lock() - .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err("fixture inference cache"))? - .iter() - .map(|(profile, results)| (profile.clone_ref(py), results.clone_ref(py))) - .collect(); - for (profile, results) in known { - if profile.bind(py).eq(inference_profile)? { - return Ok(results.into_bound(py)); - } - } - let results = PyDict::new(py); - self.by_profile - .lock() - .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err("fixture inference cache"))? - .push((inference_profile.clone().unbind(), results.clone().unbind())); - Ok(results) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/getters.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/getters.rs index 2e8ae0d008..f2f6b837c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/getters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/getters.rs @@ -3,9 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; use sqlbuild_python_base::records::main::registry::class_of::class_of; use sqlbuild_python_base::records::models::GetterEntry; +use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; /// The getter readers of this module's native pyclasses. pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs index fae540de91..1131fb370c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs @@ -1,13 +1,5 @@ -pub(crate) mod fixture_columns; -pub(crate) mod fixture_inferences; pub(crate) mod getters; pub(crate) mod model_analyses; -pub(crate) mod models; -pub(crate) mod nullability_rules; pub(crate) mod oracle; pub(crate) mod outcome_objects; -pub(crate) mod profile_inputs; -pub(crate) mod query_columns; pub(crate) mod registration; -pub(crate) mod session; -pub(crate) mod stage_functions; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 730a898710..0a6d690dd0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -7,24 +7,12 @@ use pyo3::types::{IntoPyDict, PyDict, PyDictMethods, PyFrozenSet, PyString}; use sqlbuild_analysis::assembly::analysis_session::types::Shapes; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::analysis::binding_catalog::BindingCatalog; use crate::bindings::_helpers::project_assembly::declared_relations::DeclaredShapes; +use sqlbuild_python_project::analysis::models::BindingCatalog; -use crate::bindings::_helpers::analysis_session::profile_inputs::{ - case_sensitive_shapes, nullability_rules, profile_dialect, return_types, shape_rows, -}; -use crate::bindings::_helpers::analysis_session::session::{ - CacheStatsRow, FamilyRow, ModelRow, NativeModelAnalysisSession, RequestRow, - start_model_analysis_session, -}; use crate::bindings::_helpers::compile_driver::analysis_cache::{ AnalysisCacheCounts, record_analysis_cache_metrics, }; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - AttachedSchema, NativeModelInput, attached_schema, input_analysis_sql, input_model_name, - input_reference_rows, input_sql_macro_checks, -}; -use crate::bindings::_helpers::semantic_shapes::shapes::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::cache::main::content_digest::content_digest; @@ -36,6 +24,22 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; +use sqlbuild_python_project::analysis_session::main::case_sensitive_shapes::case_sensitive_shapes; +use sqlbuild_python_project::analysis_session::main::nullability_rules::nullability_rules; +use sqlbuild_python_project::analysis_session::main::profile_dialect::profile_dialect; +use sqlbuild_python_project::analysis_session::main::return_types::return_types; +use sqlbuild_python_project::analysis_session::main::shape_rows::shape_rows; +use sqlbuild_python_project::analysis_session::main::start_model_analysis_session::start_model_analysis_session; +use sqlbuild_python_project::analysis_session::models::{ + CacheStatsRow, FamilyRow, ModelRow, NativeModelAnalysisSession, RequestRow, +}; +use sqlbuild_python_project::compiled_views::main::model_inputs::attached_schema::attached_schema; +use sqlbuild_python_project::compiled_views::main::model_inputs::input_analysis_sql::input_analysis_sql; +use sqlbuild_python_project::compiled_views::main::model_inputs::input_model_name::input_model_name; +use sqlbuild_python_project::compiled_views::main::model_inputs::input_reference_rows::input_reference_rows; +use sqlbuild_python_project::compiled_views::main::model_inputs::input_sql_macro_checks::input_sql_macro_checks; +use sqlbuild_python_project::compiled_views::models::{AttachedSchema, NativeModelInput}; +use sqlbuild_python_project::semantic_shapes::constants::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; const LOGGING_MODULE: &str = "logging"; const DEBUG_LOGGER: &str = "sqlbuild.compile"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/models.rs deleted file mode 100644 index 8fd2e82dbf..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/models.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Native state kept by the analysis-session bindings. - -use pyo3::pyclass; - -/// Fixture column inferences computed since the last reset. -#[pyclass] -#[derive(Debug, Default)] -pub(crate) struct FixtureInferenceCount { - pub(crate) computed: u64, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs index a0a1b9e965..bb49591f29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs @@ -6,12 +6,13 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::positional_record; -use crate::bindings::_helpers::lineage::lineage_facts::{confidence_member, transform_kind_member}; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::lineage::main::confidence_member::confidence_member; +use sqlbuild_python_project::lineage::main::transform_kind_member::transform_kind_member; const LINEAGE_NATIVE: &str = "native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs deleted file mode 100644 index 95e365fe7f..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs +++ /dev/null @@ -1,153 +0,0 @@ -//! Inference-profile settings crossing into native analysis: rule ids, shapes and the catalog. - -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; -use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; -use sqlbuild_host::entries::models::AdapterHost; - -use crate::bindings::_helpers::analysis::binding_catalog::{ - BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, CatalogKey, NATIVE_DIALECT_ALIASES, -}; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; -use sqlbuild_python_base::records::types::RecordId; - -const GENERIC_DIALECT: &str = "generic"; -pub(crate) const ADAPTER_RULE_ID: &str = "python"; -/// The nullability rules the native analysis knows, with their native rule ids. -const KNOWN_RULES: [(AdapterHost, &str); 3] = [ - (AdapterHost::FirstArgNullability, "first_arg"), - ( - AdapterHost::ConditionalResultNullability, - "conditional_result", - ), - (AdapterHost::SafeCastNullability, "safe_cast"), -]; - -/// The adapter's own nullability rules and `InferredNullability`, which native analysis calls. -pub(crate) type AdapterRules = Option<(Py, Py)>; - -/// The adapter nullability rules as `(name, rule id)` rows, and the adapter's own rules. -pub(crate) fn nullability_rules(profile: &Bound<'_, PyAny>) -> PyResult<(Pairs, AdapterRules)> { - let py: Python<'_> = profile.py(); - let known: Bound<'_, PyDict> = PyDict::new(py); - for (rule, id) in KNOWN_RULES { - known.set_item(host_entry(py, rule)?, id)?; - } - let adapter: Bound<'_, PyDict> = PyDict::new(py); - let mut rows: Pairs = Vec::new(); - for item in native_field(profile, "function_nullability_rules")? - .call_method0("items")? - .try_iter()? - { - let (name, rule): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; - let id: String = match known.get_item(&rule)? { - Some(id) => id.extract()?, - None => { - adapter.set_item(&name, &rule)?; - ADAPTER_RULE_ID.to_owned() - } - }; - rows.push((name.extract()?, id)); - } - if adapter.is_empty() { - return Ok((rows, None)); - } - let nullability: Bound<'_, PyAny> = record_class(py, RecordId::InferredNullability)?; - Ok((rows, Some((adapter.unbind(), nullability.unbind())))) -} - -/// `{relation: {column: value}}` as ordered rows, each value as `str(value)`. -pub(crate) fn shape_rows(shapes: &Bound<'_, PyAny>) -> PyResult { - let mut rows: Shapes = Vec::new(); - for item in mapping_items(shapes)?.try_iter()? { - let (name, shape): (String, Bound<'_, PyAny>) = item?.extract()?; - rows.push((name, text_pairs(&shape)?)); - } - Ok(rows) -} - -/// `{column: value}` as ordered pairs, each value as `str(value)`. -pub(crate) fn text_pairs(shape: &Bound<'_, PyAny>) -> PyResult { - let mut pairs: Pairs = Vec::new(); - for column in mapping_items(shape)?.try_iter()? { - let (column, value): (String, Bound<'_, PyAny>) = column?.extract()?; - pairs.push((column, value.str()?.extract()?)); - } - Ok(pairs) -} - -/// A `{key: value}` text mapping's items in order. -pub(crate) fn pairs(mapping: &Bound<'_, PyAny>) -> PyResult { - mapping_items(mapping)? - .try_iter()? - .map(|item| item?.extract()) - .collect() -} - -/// `profile.sql_analysis_dialect or "generic"`. -pub(crate) fn profile_dialect(profile: &Bound<'_, PyAny>) -> PyResult { - dialect_or_generic(&native_field(profile, "sql_analysis_dialect")?) -} - -/// `dialect or "generic"`. -pub(crate) fn dialect_or_generic(dialect: &Bound<'_, PyAny>) -> PyResult { - if dialect.is_truthy()? { - return dialect.extract(); - } - Ok(GENERIC_DIALECT.to_owned()) -} - -/// Python's `NATIVE_DIALECT_ALIASES.get(dialect, dialect)`. -pub(crate) fn aliased_dialect(_py: Python<'_>, dialect: String) -> PyResult { - Ok(NATIVE_DIALECT_ALIASES - .iter() - .find(|(alias, _)| *alias == dialect) - .map_or(dialect.clone(), |(_, native)| (*native).to_owned())) -} - -/// Python's `inferred_binding_shape` test for keeping authored identifier quoting. -pub(crate) fn case_sensitive_shapes( - profile: &Bound<'_, PyAny>, - dialect: &Bound<'_, PyAny>, -) -> PyResult { - if native_field(profile, "quoted_identifiers_ignore_case")?.is_truthy()? { - return Ok(false); - } - if dialect.is_none() { - return Ok(false); - } - let dialect: String = dialect.extract()?; - Ok(CASE_SENSITIVE_BINDING_DIALECTS.contains(&dialect.as_str())) -} - -/// `profile.binding_catalog or create_binding_catalog(...)` with no functions, types or relations. -pub(crate) fn profile_catalog<'py>( - profile: &Bound<'py, PyAny>, - dialect: &str, -) -> PyResult> { - let catalog: Bound<'py, PyAny> = native_field(profile, "binding_catalog")?; - if catalog.is_truthy()? { - return Ok(catalog); - } - let py: Python<'py> = profile.py(); - let catalog: BindingCatalog = BindingCatalog::create( - py, - CatalogKey { - dialect: dialect.to_owned(), - quoted_ignore_case: native_field(profile, "quoted_identifiers_ignore_case")? - .extract()?, - known_functions: Vec::new(), - known_types: Vec::new(), - }, - PyDict::new(py).as_any(), - )?; - Ok(Bound::new(py, catalog)?.into_any()) -} - -/// `profile.function_return_types` items in order. -pub(crate) fn return_types(profile: &Bound<'_, PyAny>) -> PyResult { - pairs(&native_field(profile, "function_return_types")?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs index aa62b2be6a..54c1e99891 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs @@ -2,17 +2,19 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use crate::bindings::_helpers::analysis_session::{ - fixture_columns, fixture_inferences, oracle, outcome_objects, query_columns, session, - stage_functions, -}; +use crate::bindings::_helpers::analysis_session::{oracle, outcome_objects}; +use sqlbuild_python_project::analysis_session::main::register_fixture_columns::register_fixture_columns; +use sqlbuild_python_project::analysis_session::main::register_query_columns::register_query_columns; +use sqlbuild_python_project::analysis_session::main::register_session::register_session; +use sqlbuild_python_project::analysis_session::main::register_stage_functions::register_stage_functions; +use sqlbuild_python_project::analysis_session::models::FixtureColumnInferences; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - session::register(module)?; - fixture_columns::register(module)?; - module.add_class::()?; + register_session(module)?; + register_fixture_columns(module)?; + module.add_class::()?; outcome_objects::register(module)?; - query_columns::register(module)?; - stage_functions::register(module)?; + register_query_columns(module)?; + register_stage_functions(module)?; oracle::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs deleted file mode 100644 index af5650a37b..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ /dev/null @@ -1,654 +0,0 @@ -//! The native model analysis session and expression-source shapes. - -use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python}; -use pyo3::types::PyDict; -use pyo3::{pyclass, pyfunction, pymethods}; -use sqlbuild_analysis::assembly::analysis_session::main::attach_analysis_cache::attach_analysis_cache; -use sqlbuild_analysis::assembly::analysis_session::main::expression_shapes::expression_shapes; -use sqlbuild_analysis::assembly::analysis_session::main::finish_analysis_session::finish_analysis_session; -use sqlbuild_analysis::assembly::analysis_session::main::finished_fact_models::finished_fact_models; -use sqlbuild_analysis::assembly::analysis_session::main::model_reference_facts::model_reference_facts; -use sqlbuild_analysis::assembly::analysis_session::main::names_set_operation::names_set_operation; -use sqlbuild_analysis::assembly::analysis_session::main::prove_dynamic_contracts::prove_dynamic_contracts; -use sqlbuild_analysis::assembly::analysis_session::main::prove_finished_dynamic_contracts::prove_finished_dynamic_contracts; -use sqlbuild_analysis::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql; -use sqlbuild_analysis::assembly::analysis_session::main::run_analysis_session::run_analysis_session; -use sqlbuild_analysis::assembly::analysis_session::main::session_sharing::session_sharing; -use sqlbuild_analysis::assembly::analysis_session::main::start_analysis_session::start_analysis_session; -use sqlbuild_analysis::assembly::analysis_session::main::take_analysis_cache::take_analysis_cache; -use sqlbuild_analysis::assembly::analysis_session::models::{ - AnalysisCacheStats, AnalysisSession, ColumnFact, DynamicFamily, ExpressionShape, - ExpressionShapeRequest, FinishedSession, LineageFacts, LineageRow, ModelOutcome, ModelRequest, - PivotBatchRequest, PivotModel, PivotOutcome, PivotTables, SessionOutcome, SessionRequest, - SessionStep, -}; -use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; -use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; -use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; -use sqlbuild_python_base::positional_record; -use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; - -use sqlbuild_cache::store::models::NativeStore; -use sqlbuild_core::constants::PANIC_MESSAGE; -use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; -use sqlbuild_core::panics::main::is_compiler_panic::is_compiler_panic; -use sqlbuild_core::panics::main::native_failure::native_failure; -use std::path::PathBuf; - -use crate::bindings::_helpers::analysis_session::nullability_rules::{ - RuleFailure, nullability_callback, raised, -}; -use crate::bindings::models::Positional; -use crate::bindings::models::ProjectCatalog; -use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; -use sqlbuild_python_base::cache::main::open_store::open_store; -use sqlbuild_python_base::cache::main::save_store::save_store; -use sqlbuild_python_base::types::CompilerDetach; - -/// `(name, type, nullability)`. -type ColumnRow = (String, Option, String); -positional_record! { - into; - /// One lineage column: output column, transform and confidence codes and its sources. - struct LineageItem { - output_column: String, - transform_code: u8, - confidence_code: u8, - /// `(resource type, resource, column)` of each source. - sources: Vec<(String, String, String)>, - } -} - -positional_record! { - extract; - /// One declared dynamic column family. - pub(crate) struct FamilyRow { - pub(crate) name: String, - pub(crate) pivot_column: String, - pub(crate) value_column: String, - pub(crate) aggregate: String, - pub(crate) data_type: String, - pub(crate) name_pattern: Option, - } -} - -impl From for FamilyRow { - fn from(family: sqlbuild_model_config::header_metadata::models::DynamicFamily) -> Self { - Self { - name: family.name, - pivot_column: family.pivot_column, - value_column: family.value_column, - aggregate: family.aggregate, - data_type: family.data_type, - name_pattern: family.name_pattern, - } - } -} - -/// A model's name, SQL, placeholders, references, lineage, required names, flags and pivot. -pub(crate) struct ModelRow { - pub(crate) name: String, - pub(crate) query_sql: String, - pub(crate) placeholders: Pairs, - pub(crate) reference_rows: Vec<(String, String)>, - pub(crate) recover_cte_facts: bool, - pub(crate) snapshot_columns: Option<(String, String)>, - pub(crate) dynamic_families: Vec, - pub(crate) sql_macro_checks: Vec, -} - -/// The profile settings, relation shapes and models one session analyses. -pub(crate) struct RequestRow { - pub(crate) dialect: String, - pub(crate) case_sensitive_shapes: bool, - pub(crate) quoted_ignore_case: bool, - pub(crate) function_return_types: Pairs, - pub(crate) known_functions: Vec, - pub(crate) nullability_rules: Option, - pub(crate) rich_type_inference: bool, - pub(crate) column_types: Shapes, - pub(crate) column_nullability: Shapes, - pub(crate) complete_schemas: Shapes, - pub(crate) catalog_schemas: Shapes, - pub(crate) dynamic_families_by_table: Vec<(String, Vec)>, - pub(crate) models: Vec, -} - -/// `(publications, failures)`. -pub(crate) type StepRow = (Shapes, Vec); - -positional_record! { - into; - /// One model's analysis, lineage source and cleaned SQL. - struct OutcomeRow { - analysis_succeeded: bool, - columns: Option>, - lineage_source: &'static str, - lineage: Vec, - has_star: bool, - star_resolved: bool, - binding_diagnostics: Vec>, - binding_validated: bool, - cleaned_sql: String, - } -} - -positional_record! { - into; - /// One dynamic pivot proof. - pub(crate) struct ProofRow { - output_proven: bool, - fixed_columns: Vec, - /// `(family, inferred type)` of each family. - families: Vec<(String, Option)>, - input_relations: Vec, - failure_reason: Option, - bare_dynamic_pivot: bool, - } -} - -/// One model's pivot SQL and declared families. -pub(crate) type PivotModelRow = (String, Vec); - -positional_record! { - extract; - /// The dialect, relation facts and families by table every proof reads, and the models. - pub(crate) struct PivotRequestRow { - pub(crate) dialect: String, - pub(crate) column_types: Shapes, - pub(crate) authoritative_types: Shapes, - pub(crate) column_nullability: Shapes, - pub(crate) families_by_table: Vec<(String, Vec)>, - pub(crate) models: Vec, - } -} - -/// `("absent" | "deferred" | "proof", proof)`. -pub(crate) type ContractRow = (&'static str, Option); - -positional_record! { - into; - /// The analysis cache's hits, misses, stored entries and why it was not read or saved. - #[derive(Clone)] - pub(crate) struct CacheStatsRow { - pub(crate) hits: usize, - pub(crate) misses: usize, - pub(crate) stored: usize, - pub(crate) failure: Option, - } -} - -positional_record! { - extract; - /// The settings and expressions one expression-source shape request infers. - pub(crate) struct ExpressionShapesRow { - pub(crate) dialect: String, - pub(crate) case_sensitive_shapes: bool, - pub(crate) function_return_types: Pairs, - pub(crate) nullability_rules: Option, - pub(crate) expressions: Vec, - } -} -/// `(catalog schema additions, analysis-shape names, whether each model's analysis succeeded)`. -pub(crate) type FinishRow = (Shapes, Vec, Vec); - -/// One compile's native model analysis; an internal failure raises `NativeCompilerError`. -#[pyclass(module = "sqlbuild._native")] -pub(crate) struct NativeModelAnalysisSession { - inner: Option, - finished: Option, - /// Each requested model's outcome and dynamic pivot proof, in request order, once finished. - outcomes: Vec<(ModelOutcome, PivotOutcome)>, - sharing: (usize, usize), - cache_path: Option, - cache_stats: Option, - rule_failure: RuleFailure, - /// The analysis dialect the session's models were marked and analysed in. - dialect: String, -} - -/// What an internal native failure of the session names. -const SESSION_CONTEXT: &str = "native model analysis"; -const SHAPES_CONTEXT: &str = "native expression-source shapes"; -/// The native store kind holding finished model analyses. -const MODEL_ANALYSIS_STORE_KIND: &str = "model-analyses"; - -impl NativeModelAnalysisSession { - /// The finished session later compile stages read, once `finish` has succeeded. - pub(crate) fn finished_session(&self) -> Option<&FinishedSession> { - self.finished.as_ref() - } - - /// Each requested model's finished outcome and dynamic pivot proof, in request order. - pub(crate) fn outcomes(&self) -> &[(ModelOutcome, PivotOutcome)] { - &self.outcomes - } - - /// The finished outcome and dynamic pivot proof of the model requested at `index`. - pub(crate) fn outcome(&self, index: usize) -> PyResult<&(ModelOutcome, PivotOutcome)> { - self.outcomes - .get(index) - .ok_or_else(|| internal_error("the session returned no analysis for a model")) - } - - /// The running session, or `NativeCompilerError` once it has failed or finished. - pub(crate) fn dialect(&self) -> &str { - &self.dialect - } - - fn running(&mut self) -> PyResult { - self.inner - .take() - .ok_or_else(|| internal_error("the session is not running")) - } -} - -/// An internal native failure of the session as `NativeCompilerError`. -fn internal_error(reason: &str) -> PyErr { - if is_compiler_panic(reason) && reason != PANIC_MESSAGE { - return compiler_error(reason); - } - compiler_error(native_failure(SESSION_CONTEXT, reason)) -} - -impl NativeModelAnalysisSession { - /// Analyse every model. - pub(crate) fn run(&mut self, py: Python<'_>) -> PyResult { - let mut session: AnalysisSession = self.running()?; - let result: Result<(AnalysisSession, SessionStep), String> = - py.compiler_detach(move || { - run_analysis_session(&mut session).map(|step| (session, step)) - }); - if let Some(error) = raised(&self.rule_failure) { - return Err(error); - } - let (session, step) = result.map_err(|reason| internal_error(&reason))?; - self.sharing = session_sharing(&session); - self.inner = Some(session); - Ok(step_row(step)) - } - - /// Every model's outcome; saves the cache best-effort. - pub(crate) fn finish(&mut self, py: Python<'_>) -> PyResult { - let mut session: AnalysisSession = self.running()?; - let cache: Option<(NativeStore, AnalysisCacheStats)> = take_analysis_cache(&mut session); - let (outcome, finished) = catch_compiler_panic(|| finish_analysis_session(session)) - .map_err(|reason| internal_error(&reason))?; - self.finished = Some(finished); - if let (Some((store, stats)), Some(path)) = (cache, self.cache_path.as_ref()) { - let saved: Option = save_store(py, &store, path, &[]) - .err() - .map(|error| error.to_string()); - self.cache_stats = Some(CacheStatsRow { - hits: stats.hits, - misses: stats.misses, - stored: stats.stored, - failure: saved, - }); - } - let SessionOutcome { - models, - schema_additions, - analysis_names, - dynamic_contracts, - } = outcome; - let succeeded: Vec = models - .iter() - .map(|model| model.analysis.analysis_succeeded) - .collect(); - self.outcomes = models.into_iter().zip(dynamic_contracts).collect(); - Ok((schema_additions, analysis_names, succeeded)) - } -} - -#[pymethods] -impl NativeModelAnalysisSession { - /// Models whose output names and lineage the finished session keeps for later stages. - #[getter] - pub(crate) fn fact_models(&self) -> Vec { - self.finished - .as_ref() - .map(finished_fact_models) - .unwrap_or_default() - } - - /// Each requested model's raw binding diagnostic rows, in request order. - pub(crate) fn binding_rows(&self) -> Vec>> { - self.outcomes - .iter() - .map(|(outcome, _)| Positional::wrap_all(outcome.analysis.binding_diagnostics.clone())) - .collect() - } - - /// The model at `index`'s outcome and proof rows, for hand-built model inputs only. - fn outcome_rows(&self, index: usize) -> PyResult<(OutcomeRow, ContractRow)> { - let (outcome, contract) = self.outcome(index)?; - let cleaned: String = resolved_cleaned_sql(outcome) - .map_err(compiler_error)? - .into_owned(); - Ok(( - outcome_row(outcome.clone(), cleaned), - contract_row(contract.clone()), - )) - } - - /// The analysis cache's `(hits, misses, stored, failure)`; None without a cache. - #[getter] - pub(crate) fn cache_stats(&self) -> Option { - self.cache_stats.clone() - } - - /// Each model's dynamic pivot proof from the finished session's tables. - pub(crate) fn prove_dynamic_contracts( - &self, - py: Python<'_>, - models: Vec, - ) -> PyResult> { - let session: &FinishedSession = self - .finished - .as_ref() - .ok_or_else(|| internal_error("the session has not finished"))?; - let models: Vec = models.into_iter().map(pivot_model).collect(); - let outcomes: Vec = py - .compiler_detach(|| prove_finished_dynamic_contracts(session, &models)) - .map_err(|reason| internal_error(&reason))?; - Ok(outcomes.into_iter().map(contract_row).collect()) - } - - /// `(shared members, shared members re-analysed alone)` as of the last step. - #[getter] - pub(crate) fn sharing(&self) -> (usize, usize) { - self.sharing - } -} - -/// Start a session on `catalog`. -pub(crate) fn start_model_analysis_session( - catalog: PyRef<'_, ProjectCatalog>, - request: RequestRow, - cache: Option<(PathBuf, String)>, - adapter_rules: Option<(Py, Py)>, -) -> PyResult { - let py: Python<'_> = catalog.py(); - let mut request: SessionRequest = - sqlbuild_python_base::compile_context::main::delta::inject_relations::inject_relations( - session_request(request), - )?; - let rule_failure: RuleFailure = RuleFailure::default(); - let dialect: String = request.dialect.clone(); - request.nullability_callback = adapter_rules - .map(|(rules, nullability)| nullability_callback(rules, nullability, rule_failure.clone())); - let catalog = &catalog.inner; - let mut session: AnalysisSession = py - .compiler_detach(|| start_analysis_session(request, catalog)) - .map_err(|reason| internal_error(&reason))?; - let cache: Option<(PathBuf, String)> = cache.filter(|_| { - sqlbuild_python_base::compile_context::main::delta::injection::injection().is_none() - }); - let mut cache_path: Option = None; - let mut cache_stats: Option = None; - if let Some((path, environment)) = cache { - match open_store(py, &path, MODEL_ANALYSIS_STORE_KIND, &environment) { - Ok(store) => { - attach_analysis_cache(&mut session, store); - cache_path = Some(path); - } - Err(error) => { - cache_stats = Some(CacheStatsRow { - hits: 0, - misses: 0, - stored: 0, - failure: Some(error.to_string()), - }); - } - } - } - Ok(NativeModelAnalysisSession { - inner: Some(session), - finished: None, - outcomes: Vec::new(), - sharing: (0, 0), - cache_path, - cache_stats, - rule_failure, - dialect, - }) -} - -/// Each expression's shape, `None` where it has none; adapter rules run as in the session. -#[pyfunction] -#[pyo3(signature = (catalog, request, adapter_rules=None))] -pub(crate) fn infer_expression_source_shapes( - py: Python<'_>, - catalog: PyRef<'_, ProjectCatalog>, - request: ExpressionShapesRow, - adapter_rules: Option<(Py, Py)>, -) -> PyResult>> { - let ExpressionShapesRow { - dialect, - case_sensitive_shapes, - function_return_types, - nullability_rules, - expressions, - } = request; - let rule_failure: RuleFailure = RuleFailure::default(); - let request = ExpressionShapeRequest { - dialect, - case_sensitive_shapes, - function_return_types, - nullability_rules, - nullability_callback: adapter_rules.map(|(rules, nullability)| { - nullability_callback(rules, nullability, rule_failure.clone()) - }), - expressions, - }; - let catalog = &catalog.inner; - let shapes = py.compiler_detach(|| expression_shapes(catalog, &request)); - if let Some(error) = raised(&rule_failure) { - return Err(error); - } - Ok(shapes - .map_err(|reason| compiler_error(native_failure(SHAPES_CONTEXT, &reason)))? - .into_iter() - .map(|shape| match shape { - ExpressionShape::Inferred(pairs) => Some(pairs), - ExpressionShape::Absent => None, - }) - .collect()) -} - -fn session_request(request: RequestRow) -> SessionRequest { - let RequestRow { - dialect, - case_sensitive_shapes, - quoted_ignore_case, - function_return_types, - known_functions, - nullability_rules, - rich_type_inference, - column_types, - column_nullability, - complete_schemas, - catalog_schemas, - dynamic_families_by_table, - models, - } = request; - SessionRequest { - dialect, - case_sensitive_shapes, - quoted_ignore_case, - function_return_types, - known_functions, - nullability_rules, - nullability_callback: None, - rich_type_inference, - column_types, - column_nullability, - complete_schemas, - catalog_schemas, - dynamic_families_by_table: families_by_table(dynamic_families_by_table), - models: models.into_iter().map(model_request).collect(), - forced_order: None, - retain_relations: sqlbuild_python_base::compile_context::main::delta::capturing::capturing( - ), - } -} - -fn model_request(row: ModelRow) -> ModelRequest { - let ModelRow { - name, - query_sql, - placeholders, - reference_rows, - recover_cte_facts, - snapshot_columns, - dynamic_families, - sql_macro_checks, - } = row; - let has_set_operation = names_set_operation(&query_sql); - let facts = model_reference_facts(&reference_rows); - let pivot_sql = if placeholders.is_empty() { - query_sql.clone() - } else { - placeholder_defaults(&query_sql, &placeholders) - }; - ModelRequest { - name, - query_sql, - placeholders, - references: facts.references, - lineage_references: facts.lineage_references, - required_names: facts.required_names, - recover_cte_facts, - has_set_operation, - snapshot_columns, - pivot_sql, - dynamic_families: dynamic_families.into_iter().map(dynamic_family).collect(), - sql_macro_checks, - } -} - -fn step_row(step: SessionStep) -> StepRow { - (step.publications, step.failures) -} - -fn column_row(column: ColumnFact) -> ColumnRow { - (column.name, column.data_type, column.nullability) -} - -fn lineage_items(rows: Vec) -> Vec { - rows.into_iter() - .map(|row| LineageItem { - output_column: row.output_column, - transform_code: row.transform_code, - confidence_code: row.confidence_code, - sources: row.sources, - }) - .collect() -} - -fn families_by_table(rows: Vec<(String, Vec)>) -> Vec<(String, Vec)> { - rows.into_iter() - .map(|(name, families)| (name, families.into_iter().map(dynamic_family).collect())) - .collect() -} - -fn dynamic_family(row: FamilyRow) -> DynamicFamily { - let FamilyRow { - name, - pivot_column, - value_column, - aggregate, - data_type, - name_pattern, - } = row; - DynamicFamily { - name, - pivot_column, - value_column, - aggregate, - data_type, - name_pattern, - } -} - -fn contract_row(outcome: PivotOutcome) -> ContractRow { - match outcome { - PivotOutcome::Absent => ("absent", None), - PivotOutcome::Proof(proof) => ( - "proof", - Some(ProofRow { - output_proven: proof.output_proven, - fixed_columns: proof.fixed_columns.into_iter().map(column_row).collect(), - families: proof.families, - input_relations: proof.input_relations, - failure_reason: proof.failure_reason, - bare_dynamic_pivot: proof.bare_dynamic_pivot, - }), - ), - } -} - -fn outcome_row(outcome: ModelOutcome, cleaned_sql: String) -> OutcomeRow { - let analysis = outcome.analysis; - let (source, lineage) = match analysis.lineage { - LineageFacts::Native(rows) => ("native", lineage_items(rows)), - LineageFacts::NativeFacts(rows) => ("facts", lineage_items(rows)), - }; - OutcomeRow { - analysis_succeeded: analysis.analysis_succeeded, - columns: analysis - .columns - .map(|columns| columns.into_iter().map(column_row).collect()), - lineage_source: source, - lineage, - has_star: analysis.has_star, - star_resolved: analysis.star_resolved, - binding_diagnostics: Positional::wrap_all(analysis.binding_diagnostics), - binding_validated: analysis.binding_validated, - cleaned_sql, - } -} - -/// Each model's dynamic pivot proof outside any session. -#[pyfunction] -pub(crate) fn prove_dynamic_column_contracts( - py: Python<'_>, - request: PivotRequestRow, -) -> PyResult> { - let PivotRequestRow { - dialect, - column_types, - authoritative_types, - column_nullability, - families_by_table: by_table, - models, - } = request; - let request = PivotBatchRequest { - tables: PivotTables { - dialect, - column_types, - authoritative_types, - column_nullability, - families_by_table: families_by_table(by_table), - }, - models: models.into_iter().map(pivot_model).collect(), - }; - let outcomes: Vec = py - .compiler_detach(|| prove_dynamic_contracts(&request)) - .map_err(|reason| internal_error(&reason))?; - Ok(outcomes.into_iter().map(contract_row).collect()) -} - -fn pivot_model(row: PivotModelRow) -> PivotModel { - let (sql, families) = row; - PivotModel { - sql, - families: families.into_iter().map(dynamic_family).collect(), - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 1fb668c0c9..8eb7797022 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -22,7 +22,6 @@ use crate::bindings::_helpers::build_progress::progress_text::write_execution_he use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; -use crate::bindings::_helpers::compiled_project::snapshot_execution::{VerifiedRun, WorkerBuild}; use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::planner_entries::command_plans::build_audit_command_plan; @@ -43,6 +42,8 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; +use sqlbuild_python_project::project_snapshot::types::VerifiedRun; const EXECUTION_COMMAND: &str = "sqb audit"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs index 5b11d2bbb2..ab6f2de2b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs @@ -1,5 +1,4 @@ mod identity; mod measurement; pub(crate) mod registration; -pub(crate) mod render; mod schema_audits; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs index 3128dee0f4..42c46345ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/registration.rs @@ -1,8 +1,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; +use sqlbuild_python_project::auditing::main::register_render::register_render; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::auditing::identity::register(module)?; crate::bindings::_helpers::auditing::measurement::register(module)?; - crate::bindings::_helpers::auditing::render::register(module)?; + register_render(module)?; crate::bindings::_helpers::auditing::schema_audits::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 8e3408df42..5faa65a20e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -19,8 +19,6 @@ use crate::bindings::_helpers::build_command::planning::{ write_compile_target, write_plan_text, }; use crate::bindings::_helpers::check::check_core::relevant_check_functions; -use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::observations::source_capture::{ arm_source_capture, take_source_capture, }; @@ -36,6 +34,8 @@ use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_use use sqlbuild_python_front::discovery::main::sources::provider_uses::provider_uses; use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; fn run_with_providers(context: &BuildContext<'_>) -> PyResult { let py: Python<'_> = context.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index dd9b0fb38d..4239e54c9f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -22,9 +22,6 @@ use crate::bindings::_helpers::build_command::policies::enforce_model_execution_ use crate::bindings::_helpers::clone::clone_pipeline::run_clone_pipeline_with_options; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::planner_scope::retained::{ScopeData, scope_data}; -use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; -use crate::bindings::_helpers::planner_selection::unit_tests::split_unit_test_selectors; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -34,6 +31,10 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_project::planner_scope::main::build_planner_scope::build_planner_scope; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::models::ScopeData; +use sqlbuild_python_project::planner_selection::main::split_unit_test_selectors::split_unit_test_selectors; const VIEW_MATERIALIZATION: &str = "view"; const RECREATED_TYPES: [&str; 2] = ["udf", "table_fn"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 59374d9439..22c526cf64 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -33,7 +33,6 @@ use crate::bindings::_helpers::consumption::recorder::{ }; use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::freshness::publication::append_freshness_records; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::observations::publication::append_build_observations; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::sql_test_command::test_command::timeout_record; @@ -60,6 +59,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; use sqlbuild_python_front::native_host::main::recorder_callbacks::recorder_callbacks; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_state::consumption::models::ChangedPolicy; const EXECUTION_COMMAND: &str = "sqb build"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index d5ae29884b..0596037169 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -14,7 +14,6 @@ use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionPro use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; -use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::build_command::main::boolean::boolean; @@ -32,6 +31,7 @@ use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::reso use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_front::discovery::models::SecretPolicy; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; const DEFER_CONFLICT_CODE: &str = "C408"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 8ab3d82fbd..c073f3eaac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -14,9 +14,6 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; -use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; -use crate::bindings::_helpers::planner_scope::loaders::build_intermediate_source_map; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -32,6 +29,9 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; +use sqlbuild_python_project::planner_scope::main::build_intermediate_source_map::build_intermediate_source_map; const SUCCESS_STATUS: &str = "success"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs index 2fda521d2e..2ca936d586 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs @@ -25,7 +25,6 @@ use crate::bindings::_helpers::build_progress::plan_facts::{BuildPlanFacts, buil use crate::bindings::_helpers::build_progress::result_text::{ function_text, load_text, model_text, optional, seed_text, test_text, text, }; -use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; @@ -48,6 +47,7 @@ use sqlbuild_python_base::build_progress::main::terminal::tty_flag::tty_flag; use sqlbuild_python_base::build_progress::main::terminal::write::write; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; const RUNTIME_LOGGER: &str = "sqlbuild.executor.runtime"; const PASS_OUTCOME: &str = "pass"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs index 34b76fdc58..7cf4d4902f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs @@ -21,13 +21,16 @@ use sqlbuild_analysis::definition_changes::models::{DefinitionSnapshot, NodeDefi use crate::bindings::_helpers::change_selection::fingerprints::{ Fingerprinting, directory_digest, file_digest, text_of, }; -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::compiled_views::source_reads; use crate::bindings::_helpers::planner_identity::metadata::{function_local_hashes, seed_identity}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; +use sqlbuild_python_project::compiled_views::main::project::sources::sources; +use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; +use sqlbuild_python_project::resource_views::main::function_reads::deps::deps; +use sqlbuild_python_project::resource_views::main::function_reads::source_file_path::source_file_path; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; /// Header values that rename rather than redefine a model. const RENAME_CONFIG_KEYS: [&str; 2] = ["migrate_from", "description"]; @@ -237,10 +240,14 @@ fn model_node( fingerprints: &Fingerprinting, uses: &BTreeMap>, ) -> PyResult<(String, NodeDefinition)> { - let name: String = model_reads::model_name(model)?; - let config: Bound<'_, PyAny> = model_reads::config(model)?; + let name: String = model_name(model)?; + let config: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::compiled_models::config::config(model)?; let config_values: Bound<'_, PyAny> = native_field(&config, "values")?; - let schema: Bound<'_, PyAny> = model_reads::schema_entry(model)?; + let schema: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( + model, + )?; let contract: String = match config_values.cast::() { Ok(values) => match values.get_item(CONTRACT_KEY)? { Some(value) => fingerprints.canonical(&value, &[])?.to_string(), @@ -296,8 +303,10 @@ fn source_node( source: &Bound<'_, PyAny>, fingerprints: &Fingerprinting, ) -> PyResult<(String, NodeDefinition)> { - let name: String = source_reads::name(source)?.extract()?; - let mut encoded: Value = fingerprints.canonical(&source_reads::source_entry(source)?, &[])?; + let name: String = + sqlbuild_python_project::resource_views::main::source_reads::name::name(source)? + .extract()?; + let mut encoded: Value = fingerprints.canonical(&source_entry(source)?, &[])?; let mut shape: Map = Map::new(); let mut reads: Option = None; if let Value::Object(fields) = &mut encoded { @@ -333,9 +342,13 @@ fn seed_node( seed: &Bound<'_, PyAny>, fingerprints: &Fingerprinting, ) -> PyResult<(String, NodeDefinition)> { - let name: String = seed_reads::name(seed)?.extract()?; + let name: String = + sqlbuild_python_project::resource_views::main::seed_reads::name::name(seed)?.extract()?; let (version_hash, _) = seed_identity(seed)?; - let schema: Bound<'_, PyAny> = seed_reads::schema_entry(seed)?; + let schema: Bound<'_, PyAny> = + sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry( + seed, + )?; let columns: String = if schema.is_none() { definition_digest(&[]) } else { @@ -450,7 +463,8 @@ fn function_nodes( fingerprints: &Fingerprinting, ) -> PyResult> { let mut nodes: Vec<(String, NodeDefinition)> = Vec::new(); - let functions: Bound<'_, PyAny> = project_reads::functions(project)?; + let functions: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)?; let hashes: Bound<'_, PyDict> = function_local_hashes( &functions, None, @@ -458,17 +472,20 @@ fn function_nodes( )?; for function in functions.try_iter()? { let function: Bound<'_, PyAny> = function?; - let key: Bound<'_, PyAny> = function_reads::key(&function)?; + let key: Bound<'_, PyAny> = + sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?; let kind: String = text_of(&native_field( &native_field(&key, "resource_type")?, "value", )?)?; - let name: String = function_reads::name(&function)?.extract()?; + let name: String = + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)? + .extract()?; let Some(hash) = hashes.get_item(&name)? else { continue; }; let mut digest: String = hash.extract()?; - let source_file: Bound<'_, PyAny> = function_reads::source_file_path(&function)?; + let source_file: Bound<'_, PyAny> = source_file_path(&function)?; if !source_file.is_none() { let path: String = text_of(&source_file)?; if path.ends_with(PYTHON_SUFFIX) { @@ -479,7 +496,7 @@ fn function_nodes( } } let mut definition: NodeDefinition = node(&kind, &name, vec![(DEFINITION_ASPECT, digest)]); - for dependency in function_reads::deps(&function)?.try_iter()? { + for dependency in deps(&function)?.try_iter()? { let dependency: Bound<'_, PyAny> = dependency?; definition.uses.push(node_key( &text_of(&native_field( @@ -501,7 +518,7 @@ fn test_file_nodes( fingerprints: &Fingerprinting, ) -> PyResult> { let mut nodes: BTreeMap = BTreeMap::new(); - for test in project_reads::sql_tests(project)?.try_iter()? { + for test in sql_tests(project)?.try_iter()? { let file: Bound<'_, PyAny> = native_field(&test?, "test_file")?; let path: String = text_of(&native_field(&file, "relative_path")?)?; let key: String = node_key(TEST_FILE_KIND, &path); @@ -590,15 +607,15 @@ pub(crate) fn definition_snapshot( ); let (connection_key, connection) = connection_node(project, discovered, &fingerprints)?; nodes.insert(connection_key, connection); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let (key, definition) = model_node(&model?, &fingerprints, &uses)?; nodes.insert(key, definition); } - for source in project_reads::sources(project)?.try_iter()? { + for source in sources(project)?.try_iter()? { let (key, definition) = source_node(&source?, &fingerprints)?; nodes.insert(key, definition); } - for seed in project_reads::seeds(project)?.try_iter()? { + for seed in seeds(project)?.try_iter()? { let (key, definition) = seed_node(&seed?, &fingerprints)?; nodes.insert(key, definition); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs index 84f252a614..293c886880 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/selection.rs @@ -26,12 +26,13 @@ use crate::bindings::_helpers::change_selection::definition_snapshot::{ declaration_uses, definition_snapshot, project_wide_keys, }; use crate::bindings::_helpers::change_selection::environment_nodes::with_current_dependencies; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::graph::keys::native_key; use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; use crate::bindings::_helpers::scenario_command::setup::{flush, write}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; const MACRO_TEST_MODE: &str = "macro"; /// The JSON document key of the changes and selection of a `--since` run. @@ -107,7 +108,7 @@ fn relation_keys((refs, sources, seeds): (&[String], &[String], &[String])) -> V /// The sources `model` reads, other than `mocked` ones, as node keys. fn read_sources(model: &Bound<'_, PyAny>, mocked: &[String]) -> PyResult> { let mut keys: Vec = Vec::new(); - for dependency in model_reads::deps(model)?.try_iter()? { + for dependency in deps(model)?.try_iter()? { let (kind, name) = native_key(&dependency?)?; if kind == SOURCE_KIND && !mocked.contains(&name) { keys.push(node_key(SOURCE_KIND, &name)); @@ -239,7 +240,7 @@ impl ChangedProject { continue; }; item.executed.extend(read_sources(model, &mocked_sources)?); - for dependency in model_reads::deps(model)?.try_iter()? { + for dependency in deps(model)?.try_iter()? { let (kind, dependency_name) = native_key(&dependency?)?; if kind == SEED_KIND && !mocked_seeds.contains(&dependency_name) { item.executed.push(node_key(SEED_KIND, &dependency_name)); @@ -328,14 +329,19 @@ pub(crate) fn tests_and_models<'py>( project: &Bound<'py, PyAny>, ) -> PyResult<(ByName<'py>, ByName<'py>)> { let mut tests: ByName<'py> = HashMap::new(); - for test in project_reads::sql_tests(project)?.try_iter()? { + for test in sql_tests(project)?.try_iter()? { let test: Bound<'py, PyAny> = test?; tests.insert(native_field(&test, "name")?.extract()?, test); } let mut models: ByName<'py> = HashMap::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model: Bound<'py, PyAny> = model?; - models.insert(model_reads::model_name(&model)?, model); + models.insert( + sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name( + &model, + )?, + model, + ); } Ok((tests, models)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 0982412eeb..14347b476b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -8,13 +8,7 @@ use crate::bindings::_helpers::compile_driver::relation_targets::build_project_r use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; -use crate::bindings::_helpers::graph::execution_gates::sql_ref_key; -use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; use crate::bindings::_helpers::planner_entries::command_plans::build_relation_command_plan; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_selection::selectors::parse_project_selector; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -36,6 +30,12 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; +use sqlbuild_python_project::graph::main::nodes::sql_ref_key::sql_ref_key; +use sqlbuild_python_project::graph::main::nodes::transitive_closure_many::transitive_closure_many; +use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_selection::main::parse_project_selector::parse_project_selector; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; use crate::bindings::_helpers::check::check_core::{ ReadSideInputs, load_results_by_loader_name, record_python_run_state_results, @@ -46,8 +46,6 @@ use crate::bindings::_helpers::check::check_results::{ }; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_outputs::runtime_target::write_python_check_runtime_target; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; -use crate::bindings::_helpers::graph::python_selection::python_node_names; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -55,6 +53,8 @@ use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; +use sqlbuild_python_project::graph::main::python_nodes::python_node_names::python_node_names; const GRAPH_OPERATOR_CODE: &str = "C681"; const GRAPH_OPERATOR_MESSAGE: &str = "sqb check selectors do not support graph operators; \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs index 15d6903129..1dce014f02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -3,8 +3,6 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; -use crate::bindings::_helpers::graph::python_selection::resolve_python_check_excludes; -use crate::bindings::_helpers::planner_phases::objects::object; use pyo3::IntoPyObject; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -12,6 +10,8 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_project::graph::main::python_nodes::resolve_python_check_excludes::resolve_python_check_excludes; +use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_core::build_command::errors::BuildCommandError; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 00ea711da8..f29dfb6eaa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -9,7 +9,6 @@ use crate::bindings::_helpers::clone::clone_command::run_clone_command; use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehouse_scope; use crate::bindings::_helpers::command_runtime::warehouses::validate_cli_warehouse; use crate::bindings::_helpers::compile_command::reuse_attempt::run_compile; -use crate::bindings::_helpers::compiled_project::job_catalog::with_job_scope; use crate::bindings::_helpers::contract_command::command::run_contract_command; use crate::bindings::_helpers::cost_command::command::run_cost_command; use crate::bindings::_helpers::dbt_command::command::run_dbt_command; @@ -57,6 +56,7 @@ use sqlbuild_python_base::cli::main::withhold_launch_record::withhold_launch_rec use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_front::discovery::models::MissingSecretsScope; +use sqlbuild_python_project::compiled_project::main::with_job_scope::with_job_scope; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index f4844e130a..4fd56ca394 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -29,7 +29,6 @@ use crate::bindings::_helpers::command_runtime::connection::resolve_target_conne use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::planner_entries::clone_plans::PhaseTimings; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::result_output::documents::format_clone_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -48,6 +47,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; const CLONE_LOGGER: &str = "sqlbuild.clone"; /// The clone phase that copies relations into the destination. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs deleted file mode 100644 index 46e83e5c42..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs +++ /dev/null @@ -1,107 +0,0 @@ -//! A compiled model's authored cursor inputs resolved into filter and watermark roles. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::pyfunction; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; - -use crate::bindings::_helpers::compiled_views::model_input_view::input_reference_rows; -use sqlbuild_project::models::ReferenceRow; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_base::records::types::RecordId; - -const CURSOR_INPUTS_KEY: &str = "cursor_inputs"; -const FILTER_ROLE: &str = "filter"; -const WATERMARK_ROLE: &str = "watermark"; -const WATERMARK_STRATEGY: &str = "watermark"; -const ROLLING_WINDOW_STRATEGY: &str = "rolling_window"; - -/// Watermark-strategy cursor inputs: each relation's column under the roles it declares. -fn role_blocks<'py>( - authored: &Bound<'py, PyDict>, - (filter, watermark): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), -) -> PyResult<()> { - for (relation, block) in authored.iter() { - let (true, Ok(block)) = ( - relation.is_instance_of::(), - block.downcast::(), - ) else { - continue; - }; - let column: Option> = block.get_item("column")?; - let roles: Option> = block.get_item("roles")?; - let (Some(column), Some(roles)) = (column, roles) else { - continue; - }; - let (true, Ok(roles)) = ( - column.is_instance_of::(), - roles.downcast::(), - ) else { - continue; - }; - if roles.contains(FILTER_ROLE)? { - filter.set_item(&relation, &column)?; - } - if roles.contains(WATERMARK_ROLE)? { - watermark.set_item(&relation, &column)?; - } - } - Ok(()) -} - -/// The string-to-string entries of an authored mapping. -fn string_map<'py>(authored: &Bound<'py, PyDict>) -> PyResult> { - let result: Bound<'py, PyDict> = PyDict::new(authored.py()); - for (key, item) in authored.iter() { - if key.is_instance_of::() && item.is_instance_of::() { - result.set_item(key, item)?; - } - } - Ok(result) -} - -/// Resolve authored cursor input fields into explicit effective roles. -#[pyfunction] -#[pyo3(signature = (*, model))] -pub(crate) fn resolve_cursor_input_roles<'py>( - model: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = model.py(); - let values: Bound<'py, PyAny> = native_field(model, "config")?.getattr("values")?; - let authored: Bound<'py, PyAny> = mapping_get(&values, CURSOR_INPUTS_KEY)?; - let strategy: Bound<'py, PyAny> = mapping_get(&values, "microbatch_strategy")?; - let rolling: bool = strategy.eq(ROLLING_WINDOW_STRATEGY)?; - let mut filter: Bound<'py, PyDict> = PyDict::new(py); - let mut watermark: Bound<'py, PyDict> = PyDict::new(py); - match authored.downcast::() { - Ok(authored) if strategy.eq(WATERMARK_STRATEGY)? => { - role_blocks(authored, (&filter, &watermark))?; - } - Ok(authored) => { - filter = string_map(authored)?; - if !rolling { - watermark = filter.clone(); - } - } - Err(_) => { - let cursor: Bound<'py, PyAny> = mapping_get(&values, "cursor")?; - if cursor.is_instance_of::() { - for ReferenceRow { name, .. } in input_reference_rows(model)? { - filter.set_item(name, &cursor)?; - } - } - if !rolling { - watermark = filter.clone(); - } - } - } - new_record_with( - py, - RecordId::CursorInputRoles, - &[ - ("filter_inputs", filter.into_any()), - ("watermark_inputs", watermark.into_any()), - ], - ) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index c6399918c0..5c3ed796fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -4,7 +4,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyTuple; -use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::cache::models::SqlTestScanCache; @@ -22,6 +21,7 @@ use sqlbuild_python_front::scopes::main::index::build_scope_lookup::build_scope_ use sqlbuild_python_front::scopes::main::index::open_native_scope_index::open_native_scope_index; use sqlbuild_python_front::sql_test_glue::main::validate_authored_sql_test_ctes::validate_authored_sql_test_ctes; use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; +use sqlbuild_python_project::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; fn compile_error(py: Python<'_>, message: Bound<'_, PyAny>) -> PyResult { Ok(PyErr::from_value( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs index a5d3007cac..61d9404905 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs @@ -1,4 +1,3 @@ -pub(crate) mod cursor_roles; pub(crate) mod declaration_scope; pub(crate) mod model_config; pub(crate) mod model_inputs; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index 0d2924508e..884976fc17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -11,8 +11,6 @@ use sqlbuild_analysis::assembly::project::main::sql_analysis_opt_out_help::sql_a use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::compile_attachment::model_config::native_model_config_session; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; use crate::bindings::_helpers::input_loops::loop_classes::model_loop_classes; use crate::bindings::_helpers::input_loops::model_loop::{LoopSwitches, NativeModelLoop}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -29,6 +27,8 @@ use sqlbuild_python_front::model_config::main::model_nodes::unicode_environment: use sqlbuild_python_front::model_config::models::NativeModelValidator; use sqlbuild_python_front::render::main::expansion::macro_expansion_classes::macro_expansion_classes; use sqlbuild_python_front::render::models::ConstantRendering; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_views::models::NativeModelInput; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs index a6aa6e901d..7770b938b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; use crate::bindings::_helpers::compile_attachment::{ - cursor_roles, declaration_scope, model_config, model_inputs, sql_expansion_context, target, + declaration_scope, model_config, model_inputs, sql_expansion_context, target, }; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config_function::build_effective_connection_config_function; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_function::build_effective_connection_function; @@ -13,6 +13,7 @@ use sqlbuild_python_front::compile_attachment::main::build_effective_settings_co use sqlbuild_python_front::compile_attachment::main::build_effective_settings_function::build_effective_settings_function; use sqlbuild_python_front::compile_attachment::main::build_effective_target_config_function::build_effective_target_config_function; use sqlbuild_python_front::compile_attachment::main::build_effective_vars_function::build_effective_vars_function; +use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles_function::resolve_cursor_input_roles_function; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!( @@ -47,9 +48,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { declaration_scope::rebind_declaration_scope, module )?)?; - module.add_function(wrap_pyfunction!( - cursor_roles::resolve_cursor_input_roles, - module - )?)?; + module.add_function(resolve_cursor_input_roles_function(module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index 8ce8ee5894..5fdcc2c834 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -14,7 +14,6 @@ use sqlbuild_core::cli_text::models::{ use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::compile_command::surrogates::SurrogateText; -use crate::bindings::_helpers::semantic_shapes::coverage::semantic_coverage; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::compile_outputs::main::emit_json_report::emit_json_report; @@ -28,6 +27,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::discovery::constants::SQL_HOOK_OUTPUT_FIELDS; use sqlbuild_python_front::discovery::main::headers::hook_entries::hook_entries; +use sqlbuild_python_project::semantic_shapes::main::semantic_coverage::semantic_coverage; const METADATA_MODULE: &str = "importlib.metadata"; const TARGET_DIRECTORY: &str = "target"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 27c9fe1a28..ddd0e533d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -33,11 +33,6 @@ use crate::bindings::_helpers::compile_outputs::artifacts::{ use crate::bindings::_helpers::compile_outputs::runtime_target::{ compiled_sql_test_path, planned_sql_test_path, }; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::sql_test_glue::models::SqlTestPlanOptions; -use crate::bindings::_helpers::sql_test_glue::plan_driver::planned_rows; -use crate::bindings::_helpers::sql_test_glue::plan_driver::sql_test_model_chain_names_by_key; -use crate::bindings::_helpers::sql_test_glue::planning::PlanRow; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -56,6 +51,11 @@ use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::path_posix_text::path_posix_text; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; +use sqlbuild_python_project::sql_test_glue::main::planned_rows::planned_rows; +use sqlbuild_python_project::sql_test_glue::main::sql_test_model_chain_names_by_key::sql_test_model_chain_names_by_key; +use sqlbuild_python_project::sql_test_glue::models::PlanRow; +use sqlbuild_python_project::sql_test_glue::models::SqlTestPlanOptions; const COMPILED_DIR: &str = "compiled"; const MODELS_DIR: &str = "models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs index 125c22a667..d98c4c3381 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/report_hooks.rs @@ -7,7 +7,6 @@ use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple, PyTupleMethod use sqlbuild_core::cli_text::models::ModelLine; use crate::bindings::_helpers::compile_command::report::diagnostic_item; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; use sqlbuild_python_base::compile_context::main::delta::capture::capture; use sqlbuild_python_base::compile_context::main::delta::capturing::capturing; use sqlbuild_python_base::compile_context::main::delta::guarded::guarded; @@ -17,6 +16,7 @@ use sqlbuild_python_base::compile_context::models::CapturedModel; use sqlbuild_python_base::compile_context::models::CapturedTest; use sqlbuild_python_base::compile_outputs::main::plain_json::plain_json; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::graph::models::NativeProjectGraph; /// The graph key kind of a model. const MODEL_KIND: &str = "model"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 1a127cd3fa..b7ce0c6d93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -10,10 +10,7 @@ use sqlbuild_rules::lint_text::main::counts_toward_exit::counts_toward_exit; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph_with_analysis_selection; -use crate::bindings::_helpers::contracts::contract_diagnostics::evaluate_model_contracts; use crate::bindings::_helpers::lineage::project_lineage::build_project_column_lineage; -use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::rules::constants::RULE_FIX_AVAILABLE_NOTE; use crate::bindings::_helpers::rules::early_builtin_rules::{EarlyRulesScope, EarlyRulesSettings}; use crate::bindings::_helpers::rules::early_sql_lint::EarlySqlLint; @@ -36,6 +33,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_project::contracts::main::evaluate_model_contracts::evaluate_model_contracts; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; +use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; const RICH_LINEAGE: &str = "rich"; const FAST_LINEAGE: &str = "fast"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index 621adc4833..78d31418c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -11,9 +11,9 @@ use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_python_base::compile_context::main::diagnostic_collector::collect_compile_diagnostics::collect_compile_diagnostics; use crate::bindings::_helpers::compile_driver::assembly_project::assembled_project; use crate::bindings::_helpers::compile_driver::project::lineage_mode; -use crate::bindings::_helpers::compiled_views::records::project_with; -use crate::bindings::_helpers::compiled_views::records::record_model_analyses; -use crate::bindings::_helpers::references::explicit_references::explicit_reference_report; +use sqlbuild_python_project::compiled_views::main::project::project_with::project_with; +use sqlbuild_python_project::compiled_views::main::project::record_model_analyses::record_model_analyses; +use sqlbuild_python_project::references::main::explicit_reference_report::explicit_reference_report; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index d74df6da1c..774472d267 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -13,24 +13,14 @@ use sqlbuild_host::compile_context::main::record_compile_cpu_timing::record_comp use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::analysis::binding_catalog::{ - BindingCatalog, CatalogKey, ignores_quoted_case, -}; use crate::bindings::_helpers::analysis_session::model_analyses::{ ModelAnalyses, ModelAnalysisRequest, analyze_models, }; -use crate::bindings::_helpers::analysis_session::profile_inputs::{ - ADAPTER_RULE_ID, nullability_rules, text_pairs, -}; -use crate::bindings::_helpers::analysis_session::stage_functions::expression_source_shapes; use crate::bindings::_helpers::compile_driver::analysis_cache::analysis_cache_context; use crate::bindings::_helpers::compile_driver::assembly::{AssemblyRequest, AssemblyStage}; -use crate::bindings::_helpers::compiled_views::model_input_view::attached_schema; -use crate::bindings::_helpers::compiled_views::source_views::read_source_view; use crate::bindings::_helpers::project_assembly::declared_relations::{ DeclaredRelations, DeclaredShapes, declared_relations, }; -use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::compile_context::models::PythonProcessClock; @@ -48,6 +38,16 @@ use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::sqltext::main::optional_record_syntax::optional_record_syntax; +use sqlbuild_python_project::analysis::main::ignores_quoted_case::ignores_quoted_case; +use sqlbuild_python_project::analysis::models::{BindingCatalog, CatalogKey}; +use sqlbuild_python_project::analysis_session::constants::ADAPTER_RULE_ID; +use sqlbuild_python_project::analysis_session::main::expression_source_shapes::expression_source_shapes; +use sqlbuild_python_project::analysis_session::main::nullability_rules::nullability_rules; +use sqlbuild_python_project::analysis_session::main::text_pairs::text_pairs; +use sqlbuild_python_project::compiled_views::main::model_inputs::attached_schema::attached_schema; +use sqlbuild_python_project::resource_views::main::source_views::read_source_view::read_source_view; +use sqlbuild_python_project::semantic_shapes::main::declared_types::declared_types; +use sqlbuild_python_project::semantic_shapes::main::function_names::function_names; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use std::collections::HashMap; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs index 60ef674765..22b0b991b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -4,10 +4,6 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use crate::bindings::_helpers::analysis_session::profile_inputs::dialect_or_generic; -use crate::bindings::_helpers::analysis_session::session::{ - FamilyRow, NativeModelAnalysisSession, PivotRequestRow, prove_dynamic_column_contracts, -}; use crate::bindings::_helpers::compile_driver::assembly_analysis::AnalysisContext; use crate::bindings::_helpers::project_assembly::models::{NativeModelFacts, NativeProjectFacts}; use crate::bindings::_helpers::project_assembly::native_resources::assemble_native_project_resources; @@ -22,6 +18,11 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::models::HeaderFamilies; +use sqlbuild_python_project::analysis_session::main::dialect_or_generic::dialect_or_generic; +use sqlbuild_python_project::analysis_session::main::prove_dynamic_column_contracts::prove_dynamic_column_contracts; +use sqlbuild_python_project::analysis_session::models::{ + FamilyRow, NativeModelAnalysisSession, PivotRequestRow, +}; /// `NativeProjectFacts` of `inputs`: resources, syntax validity, deps and standalone pivot proofs. pub(crate) fn native_project_facts<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 1008452f1f..4b396dea53 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -8,14 +8,9 @@ use sqlbuild_analysis::analysis_route::main::analysis_route::analysis_route; use sqlbuild_analysis::analysis_route::models::AnalysisRoute; use crate::bindings::_helpers::analysis_session::model_analyses::ModelAnalyses; -use crate::bindings::_helpers::analysis_session::profile_inputs::profile_dialect; -use crate::bindings::_helpers::analysis_session::query_columns::ColumnsRow; -use crate::bindings::_helpers::analysis_session::stage_functions::query_column_rows; use crate::bindings::_helpers::compile_driver::assembly_analysis::{ AnalysisContext, model_stem, selected_for_analysis, }; -use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_views::model_input_view::input_analysis_sql; use crate::bindings::_helpers::hand_built::compiled_model::compiled_model as hand_built_model; use crate::bindings::_helpers::project_assembly::syntax_checks::model_placeholders; use crate::bindings::_helpers::project_assembly::syntax_validation::{ @@ -28,6 +23,11 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; use sqlbuild_python_front::render::main::calls::top_level_macro_call_names::top_level_macro_call_names; +use sqlbuild_python_project::analysis_session::main::profile_dialect::profile_dialect; +use sqlbuild_python_project::analysis_session::main::query_column_rows::query_column_rows; +use sqlbuild_python_project::analysis_session::types::ColumnsRow; +use sqlbuild_python_project::compiled_views::main::model_inputs::input_analysis_sql::input_analysis_sql; +use sqlbuild_python_project::compiled_views::models::NativeModelInput; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 089b7861b2..c5811e1da4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -24,7 +24,6 @@ use crate::bindings::_helpers::compile_driver::assembly_models::{ binding_rows, compiled_models, project_binding_diagnostics, }; use crate::bindings::_helpers::compile_driver::assembly_sql_tests::assembled_sql_tests; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::semantic_checks::stage::{ SemanticCompletion, completed_semantic_project, }; @@ -43,6 +42,7 @@ use sqlbuild_python_front::contracts::main::resolve_effective_scenario_config::r use sqlbuild_python_front::model_config::main::config::early_model_templates::early_model_templates; use sqlbuild_python_front::scopes::main::checks::scope_index_with_compile_usages::scope_index_with_compile_usages; use sqlbuild_python_front::scopes::main::reports::report_scope_index_errors::report_scope_index_errors; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; const SOURCE_SLOT: &str = "source"; const SEED_SLOT: &str = "seed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs index e2fe06bb1b..9ae7596cda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs @@ -13,8 +13,6 @@ use sqlbuild_analysis::diff_selection::main::resolve_diff_selection::resolve_dif use sqlbuild_analysis::diff_selection::models::DiffGraph; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::graph::views::project_lineage_views; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; @@ -23,6 +21,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::graph::main::project::project_lineage_views::project_lineage_views; /// Indexes of the `CompiledObjectKey`s a diff graph names, assigned on first sight. struct KeyIndexes<'py> { @@ -166,13 +167,13 @@ fn diff_model_names<'py>( let mut names: Vec> = Vec::new(); for model in native_field(project, "models")?.try_iter()? { let model: Bound<'py, PyAny> = model?; - let key: Bound<'py, PyAny> = model_reads::model_key(&model)?; + let key: Bound<'py, PyAny> = model_key(&model)?; let chosen: bool = match keys.indexes.get_item(&key)? { Some(index) => selected.contains(&index.extract::()?), None => false, }; if chosen && native_field(&key, "resource_type")?.eq(&model_type)? { - names.push(model_reads::name_object(&model)?); + names.push(name_object(&model)?); } } PyTuple::new(py, names) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 5d97463a57..a1ae3cec86 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -22,9 +22,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::compile_driver::main::reject_invalid_project_configuration::reject_invalid_project_configuration; use sqlbuild_python_front::compile_driver::models::ProjectConfiguration; -use sqlbuild_python_front::discovery::constants::ALLOW_MISSING_SECRETS_ENV; -use sqlbuild_python_front::discovery::constants::ALLOW_MISSING_SECRETS_SETTING; -use sqlbuild_python_front::discovery::constants::TRUE_ENV_VALUES; +use sqlbuild_python_front::discovery::constants::{ + ALLOW_MISSING_SECRETS_ENV, ALLOW_MISSING_SECRETS_SETTING, TRUE_ENV_VALUES, +}; use sqlbuild_python_front::discovery::main::project::load_local_config_file::load_local_config_file; use sqlbuild_python_front::discovery::main::project::load_project_config_file::load_project_config_file; use sqlbuild_python_front::discovery::main::project::warn_deprecated_settings::warn_deprecated_settings; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index 6bad877a8f..16dbc2b370 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -11,8 +11,9 @@ use sqlbuild_python_front::compile_attachment::main::build_effective_settings::b use sqlbuild_python_front::compile_attachment::main::build_effective_vars::build_effective_vars; use crate::bindings::_helpers::compile_attachment::model_inputs::build_model_inputs; use sqlbuild_python_base::compile_context::main::diagnostic_collector::collect_compile_diagnostics::collect_compile_diagnostics; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::records::{compile_inputs_view, inputs_with}; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_views::main::project::compile_inputs_view::compile_inputs_view; +use sqlbuild_python_project::compiled_views::main::project::inputs_with::inputs_with; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use crate::bindings::_helpers::input_loops::attachment::{attach_checks, attach_resources}; use sqlbuild_python_front::model_config::main::config::run_id::run_id; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index e2f94f9638..a43af45a37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -10,7 +10,6 @@ use crate::bindings::_helpers::compile_command::target_writer::{ Prepared, planning_diagnostics, write_static_target, }; use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::manifest::builder::build_manifest; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -23,6 +22,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; const EMPTY_DAG_PATH: &str = ""; const TARGET_DIRECTORY: &str = "target"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs index fccf7712f3..c2ca68e69e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs @@ -5,13 +5,13 @@ use pyo3::types::{PyDict, PyFloat}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::pipeline::compile_project_phase; -use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::graph::models::{CompiledProjectGraph, NativeProjectGraph}; /// Discovered Python node functions whose selectors need the Python graph. const PYTHON_NODE_FUNCTIONS: [&str; 4] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 6ab72cb494..7cdd47a44b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -9,13 +9,14 @@ use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_g use crate::bindings::_helpers::compile_driver::python_plan_entries::{ build_python_run_plan_outputs, python_read_source_names_for, }; -use crate::bindings::_helpers::compiled_views::{project_reads, source_reads}; -use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; use crate::bindings::_helpers::planner_entries::execution_plan::build_execution_plan; -use crate::bindings::_helpers::planner_phases::objects::object; use sqlbuild_analysis::planner_scope::constants::SOURCE; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::graph::models::KeyCache; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; use std::collections::HashSet; use pyo3::exceptions::PyException; @@ -24,12 +25,12 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::phases::monotonic; -use crate::bindings::_helpers::graph::python_node_graph::run_selection_from_inputs; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::graph::main::python_nodes::run_selection_from_inputs::run_selection_from_inputs; const SINGLE_CONNECTION: i64 = 1; @@ -67,12 +68,17 @@ fn without_loaders<'py>( } } let mut sources: Vec<(String, String)> = Vec::new(); - for source in project_reads::sources(project)?.try_iter()? { + for source in sqlbuild_python_project::compiled_views::main::project::sources::sources(project)? + .try_iter()? + { let source = source?; - let loader: Option = - native_field(&source_reads::source_entry(&source)?, "loader")?.extract()?; + let loader: Option = native_field(&source_entry(&source)?, "loader")?.extract()?; if loader.is_some_and(|loader| dropped.contains(&loader)) { - sources.push((SOURCE.to_owned(), source_reads::name(&source)?.extract()?)); + sources.push(( + SOURCE.to_owned(), + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)? + .extract()?, + )); } } let mut selected: Vec<(String, String)> = sql_keys diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 488081bd09..421fab6a16 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -30,8 +30,6 @@ use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::compile_driver::assembly::assemble_project; use crate::bindings::_helpers::compile_driver::selection_phases::resolve_analysis_model_names; -use crate::bindings::_helpers::compiled_views::records::project_with; -use crate::bindings::_helpers::contracts::contract_diagnostics::promotion_conflict_diagnostics; use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_core::constants::TABLE_PROMOTION_MODE_SETTING_KEY; @@ -43,6 +41,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::contracts::main::resolve_effective_collection_rendering::resolve_effective_collection_rendering; +use sqlbuild_python_project::compiled_views::main::project::project_with::project_with; +use sqlbuild_python_project::contracts::main::promotion_conflict_diagnostics::promotion_conflict_diagnostics; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; /// Optional keywords of the project graph builders. const GRAPH_OPTIONAL: [&str; 7] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs index e58b34c785..cdd6b92def 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -2,11 +2,11 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::replaced::replaced; +use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; use std::collections::HashMap; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -16,7 +16,6 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::python_plan::main::ordered_python_names::ordered_python_names; use sqlbuild_analysis::python_plan::main::upstream_closure::upstream_closure; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -24,6 +23,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; const SKIPPED_UPSTREAM_CODE: &str = "P501"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs index 49b36814fd..b137970c1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs @@ -7,13 +7,13 @@ use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::{AdapterHost, UserCodeHost}; -use crate::bindings::_helpers::references::source_relations::render_source_relation; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index 85d6a569ae..005212247d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -9,14 +9,15 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::assembly::assemble_project; use crate::bindings::_helpers::compile_driver::pipeline_entries::compile_analysis_selection; use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; -use crate::bindings::_helpers::planner_scope::retained::{ScopeData, scope_data}; -use crate::bindings::_helpers::planner_selection::unit_tests::split_project_unit_test_selectors; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::models::ScopeData; +use sqlbuild_python_project::planner_selection::main::split_project_unit_test_selectors::split_project_unit_test_selectors; const DATACLASSES_MODULE: &str = "dataclasses"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs index 6999a8412b..12cc3c0e8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs @@ -10,7 +10,6 @@ use crate::bindings::_helpers::compile_driver::deferral::{ use crate::bindings::_helpers::compile_driver::pipeline::compile_project_phase; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; use crate::bindings::_helpers::compile_driver::selection_phases::unit_test_selectors_applied; -use crate::bindings::_helpers::graph::python_node_graph::run_selection_from_inputs; use crate::bindings::_helpers::planner_entries::command_plans::{ resolve_static_command_scope, resolve_static_relation_context, }; @@ -19,6 +18,7 @@ use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::graph::main::python_nodes::run_selection_from_inputs::run_selection_from_inputs; /// Locations of the deferred target's relations, when the options defer to one. fn deferred_locations<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs index 06535472a5..149855f344 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs @@ -3,16 +3,16 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; -use crate::bindings::_helpers::compiled_views::records::project_with; -use crate::bindings::_helpers::compiled_views::seed_views::NativeCompiledSeed; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::project::project_with::project_with; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; +use sqlbuild_python_project::resource_views::models::NativeCompiledFunction; +use sqlbuild_python_project::resource_views::models::NativeCompiledSeed; /// The adapter defaults every compiled location falls back to. pub(crate) struct TargetDefaults<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs deleted file mode 100644 index 4f548c9a9f..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs +++ /dev/null @@ -1,298 +0,0 @@ -//! The resource rows and settings the compile attaches into the project, and the installed runs. - -use sqlbuild_python_base::positional_record; -use std::collections::HashMap; -use std::sync::OnceLock; - -use pyo3::FromPyObject; -use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; -use pyo3::types::{PyDict, PyString}; - -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::audit_views::NativeAuditInput; -use crate::bindings::_helpers::compiled_views::function_views::NativeFunctionInput; -use crate::bindings::_helpers::compiled_views::source_views::NativeSourceInput; -use sqlbuild_python_base::cache::models::SqlTestScanStore; - -const ATTACHMENT_UNINSTALLED: &str = "the native module registered no resource attachment"; - -static INSTALLED: OnceLock = OnceLock::new(); - -/// One discovered source: its file's path, the discovered file and the entry. -pub(crate) type SourceRow = (String, Py, Py); - -/// `(default contract, whether expressions are syntax-checked, source kind)`. -pub(crate) type SourceSettings = (Option, bool, Py); - -positional_record! { - extract; - /// A function's paths, stem, discovered file, header values, body and SQL file contents. - pub(crate) struct FunctionRow { - pub(crate) file_path: String, - pub(crate) relative_path: String, - pub(crate) name: String, - pub(crate) function_file: Py, - pub(crate) header_values: Py, - pub(crate) body: String, - pub(crate) contents: Option, - } -} - -positional_record! { - extract; - /// Adapter, validation, raw default and target namespaces, inheritance and resource kind. - pub(crate) struct FunctionSettings { - pub(crate) adapter_name: String, - pub(crate) validate: bool, - pub(crate) defaults: (Option, Option), - pub(crate) target: Option<(Option, Option)>, - pub(crate) inherit: bool, - pub(crate) kind: Py, - } -} - -/// A test case's discovered case, name and parameter values in authored order. -pub(crate) type TestCaseRow = (Py, String, Vec<(String, Py)>); - -positional_record! { - extract; - /// A test block, its name, SQL, mode, declared parameter names, cases and authored SQL span. - pub(crate) struct TestBlockRow { - pub(crate) block: Py, - pub(crate) name: Option, - pub(crate) sql_body: String, - pub(crate) mode: String, - pub(crate) parameters: Vec, - pub(crate) cases: Vec, - pub(crate) sql_body_span: Option<(usize, usize)>, - } -} - -positional_record! { - extract; - /// A test file's paths, stem, discovered file, contents and blocks. - pub(crate) struct TestFileRow { - pub(crate) file_path: String, - pub(crate) relative_path: String, - pub(crate) stem: String, - pub(crate) file: Py, - pub(crate) contents: String, - pub(crate) blocks: Vec, - } -} - -/// What every SQL test and scenario of one compile resolves and records with. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct SqlTestSettings { - /// `CompileSqlTestInput`, its model and direct payloads, and `CompileSqlTestCte`. - pub(crate) test_input: Py, - pub(crate) model_payload: Py, - pub(crate) direct_payload: Py, - pub(crate) test_cte: Py, - /// `CompileSqlScenarioInput` and `CompileSqlScenarioCte`. - pub(crate) scenario_input: Py, - pub(crate) scenario_cte: Py, - /// `SqlTestMode` by value. - pub(crate) modes: Py, - /// The `ResourceKind`s of tests and scenarios, and the runtime `UsageKind`. - pub(crate) test_kind: Py, - pub(crate) scenario_kind: Py, - pub(crate) runtime_usage: Py, - /// The adapter's typed scalar renderer, returning `(text, None)` or `(None, error)`. - pub(crate) render_parameter: Py, - pub(crate) adapter_name: String, - /// `SourceLocation`, for helper problems located natively. - pub(crate) source_location: Py, - /// The shared scan store with the scan algorithm and lexical rules keying it, if any. - pub(crate) scan_store: Option>, - pub(crate) scan_key: (String, String), -} - -/// The attached SQL test inputs, then scenario inputs. -pub(crate) type AttachedSqlTests = (Vec>, Vec>); - -positional_record! { - extract; - /// A scenario file's paths, discovered file, contents, SQL body and name. - pub(crate) struct ScenarioRow { - pub(crate) file_path: String, - pub(crate) relative_path: String, - pub(crate) scenario_file: Py, - pub(crate) contents: String, - pub(crate) sql_body: String, - pub(crate) name: String, - } -} - -positional_record! { - extract, into; - /// An audit block, its name, SQL body, evidence SQL, measurement flag and header values. - pub(crate) struct AuditBlockRow { - pub(crate) block: Py, - pub(crate) name: Option, - pub(crate) sql_body: String, - pub(crate) evidence_sql: Option, - pub(crate) measurement: bool, - pub(crate) header: Py, - } -} - -positional_record! { - extract, into; - /// An audit file's paths, stem, discovered file, generic flag, contents and blocks. - pub(crate) struct AuditFileRow { - pub(crate) file_path: String, - pub(crate) relative_path: String, - pub(crate) stem: String, - pub(crate) file: Py, - pub(crate) generic: bool, - pub(crate) contents: String, - pub(crate) blocks: Vec, - } -} - -/// What every audit in one compile resolves against, and the Python values it builds with. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct AuditSettings { - pub(crate) default_severity: Option, - pub(crate) default_run_scope: Option, - /// Built-in generic definitions as audit file rows, overridden by project definitions. - pub(crate) builtins: Vec, - /// `DeclarationIdentity`, and the singular and generic audit `DeclarationKind`s. - pub(crate) declaration_identity: Py, - pub(crate) singular_kind: Py, - pub(crate) audit_kind: Py, - /// `ResourceKind` by value, for the resources singular audits read and attach to. - pub(crate) resource_kinds: Py, -} - -/// A loader's description, relative path and whether that path is a YAML file. -pub(crate) type LoaderRow = (Option, String, bool); - -positional_record! { - extract; - /// A Python resource's kind, name, description (a `str` that may hold surrogates), location, help. - pub(crate) struct PythonResourceRow { - pub(crate) kind: String, - pub(crate) name: String, - pub(crate) description: Option>, - pub(crate) location: Py, - pub(crate) help: String, - } -} - -positional_record! { - extract; - /// A scenario's name, relative path, contents and authored header description. - pub(crate) struct ScenarioDescriptionRow { - pub(crate) name: String, - pub(crate) path: String, - pub(crate) contents: String, - pub(crate) raw: Py, - } -} - -positional_record! { - extract; - /// A SQL hook file's name, relative path, contents and description. - pub(crate) struct HookDescriptionRow { - pub(crate) name: String, - pub(crate) path: String, - pub(crate) contents: String, - pub(crate) description: Option, - } -} - -/// What the resource diagnostics read beyond the owner, and the Python values they build. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct DiagnosticSettings { - /// `CompilerDiagnostic`, `SourceLocation`, `Path` and `CompileInputError`. - pub(crate) compiler_diagnostic: Py, - pub(crate) compile_error: Py, - pub(crate) source_location: Py, - pub(crate) path_class: Py, - /// Every discovered model file, whose headers attach audit factories. - pub(crate) model_files: Vec>, - /// Each discovered audit factory's name, relative path and line. - pub(crate) audit_factories: Vec<(String, String, usize)>, - /// Each scenario's name, relative path, contents and authored header description. - pub(crate) scenarios: Vec, - /// Each SQL hook file's name, relative path, contents and description. - pub(crate) sql_hooks: Vec, - /// Each loader by name. - pub(crate) loaders: HashMap, - /// Python-defined resources: kind, name, description, location and help, read by Python. - pub(crate) python_resources: Vec, - /// The project's `max_description_lines` and `line_width`, which Python descriptions keep to. - pub(crate) description_limit: (usize, i64), -} - -/// The orphaned audit factory warnings, then the missing descriptions, then the cursor errors. -pub(crate) type ResourceDiagnostics<'py> = ( - Vec>, - Vec>, - Vec>, -); - -/// Attaches the project's sources, given their rows and settings. -pub(crate) type AttachSources = fn( - &Bound<'_, NativeCompiledProject>, - Vec, - SourceSettings, -) -> PyResult>; - -/// Attaches the project's SQL, then Python, functions. -pub(crate) type AttachFunctions = fn( - &Bound<'_, NativeCompiledProject>, - (Vec, Vec), - FunctionSettings, -) -> PyResult>; - -/// Reads the project's resource diagnostics. -pub(crate) type ResourceDiagnosticsRun = for<'py> fn( - &NativeCompiledProject, - Python<'py>, - &DiagnosticSettings, -) -> PyResult>; - -/// Attaches the project's SQL test files, then its scenarios. -pub(crate) type AttachSqlTests = fn( - &Bound<'_, NativeCompiledProject>, - Vec, - Vec, - SqlTestSettings, -) -> PyResult; - -/// Attaches the project's audits; returns their views and the built-in shadow warnings. -pub(crate) type AttachAudits = - for<'py> fn( - &Bound<'py, NativeCompiledProject>, - Vec, - AuditSettings, - ) -> PyResult<(Vec, Vec>)>; - -/// The compile's runs behind the project's attachment methods. -#[derive(Clone, Copy, Debug)] -pub(crate) struct ResourceAttachment { - pub(crate) sources: AttachSources, - pub(crate) functions: AttachFunctions, - pub(crate) diagnostics: ResourceDiagnosticsRun, - pub(crate) sql_tests: AttachSqlTests, - pub(crate) audits: AttachAudits, -} - -/// Install the compile's attachment runs once, at registration. -pub(crate) fn install(attachment: ResourceAttachment) { - let _ = INSTALLED.get_or_init(|| attachment); -} - -/// The installed attachment runs. -pub(crate) fn resource_attachment() -> PyResult<&'static ResourceAttachment> { - INSTALLED - .get() - .ok_or_else(|| PyRuntimeError::new_err(ATTACHMENT_UNINSTALLED)) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 8f4b12ec83..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -1,16 +1 @@ -pub(crate) mod attachment_inputs; -pub(crate) mod constants; -pub(crate) mod job_catalog; -pub(crate) mod project; pub(crate) mod registration; -pub(crate) mod rows; -pub(crate) mod schema_facts; -pub(crate) mod snapshot; -pub(crate) mod snapshot_boundary; -pub(crate) mod snapshot_declarations; -pub(crate) mod snapshot_execution; -pub(crate) mod snapshot_models; -pub(crate) mod snapshot_resources; -pub(crate) mod snapshot_roles; -pub(crate) mod snapshot_worker; -pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs deleted file mode 100644 index 6f644d93cc..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ /dev/null @@ -1,664 +0,0 @@ -//! The compile's natively owned project, which producers record into and consumers read. - -use sqlbuild_python_base::positional_record; -use std::collections::{BTreeMap, HashMap}; -use std::sync::Mutex; - -use pyo3::exceptions::{PyIndexError, PyRuntimeError}; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyString}; -use pyo3::{FromPyObject, pyclass, pymethods}; -use serde_json::Value; -use sqlbuild_analysis::compiled_project::main::project_models::project_models; -use sqlbuild_analysis::compiled_project::main::record_model_analysis::record_model_analysis; -use sqlbuild_analysis::compiled_project::main::record_skipped_analysis::record_skipped_analysis; -use sqlbuild_analysis::compiled_project::models::{ - CompiledModelFacts, CompiledProjectFacts, DeclarationFact, DynamicProofFact, ModelAnalysisFact, - ModelConfigFacts, ModelReferenceFact, ModelSchemaFact, -}; - -use sqlbuild_project::models::CompiledProject; - -use crate::bindings::_helpers::compiled_project::attachment_inputs::AttachedSqlTests; -use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditFileRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::DiagnosticSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::FunctionRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::FunctionSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::ResourceDiagnostics; -use crate::bindings::_helpers::compiled_project::attachment_inputs::ScenarioRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::SourceRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::SourceSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::SqlTestSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::TestFileRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::resource_attachment; -use crate::bindings::_helpers::compiled_project::values::{object_value, plain_value}; -use crate::bindings::_helpers::compiled_views::audit_views::{ - AttachedAudit, AuditRecord, NativeAuditInput, NativeCompiledAudit, audit_record, -}; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - CompiledViewClasses, NativeCompiledModel, -}; -use crate::bindings::_helpers::compiled_views::function_views::{ - AttachedFunction, FunctionRecord, NativeCompiledFunction, NativeFunctionInput, function_record, -}; -use crate::bindings::_helpers::compiled_views::inputs_view::NativeInputsView; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - ModelInputRecord, NativeModelInput, -}; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::compiled_views::seed_views::{ - NativeCompiledSeed, NativeSeedInput, SeedNamespace, SeedRecord, SeedViews, paired_seeds, -}; -use crate::bindings::_helpers::compiled_views::source_views::{ - NativeCompiledSource, NativeSourceInput, SourceRecord, -}; -use sqlbuild_project::types::InferredRow; -use sqlbuild_project::types::ObjectKeyRow; -use sqlbuild_project::types::SourceNamespace; -use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; -use sqlbuild_python_front::render::models::ConstantRendering; - -pub(crate) const CONTRACT_KEY: &str = "contract"; -const INPUT_CLASSES_UNSET: &str = "no model loop recorded inputs into this project"; -const ATTACHMENT_LOOP_UNSET: &str = "no model loop was kept to attach this project's resources"; -pub(crate) const INCREMENTAL_MODE_KEY: &str = "incremental_mode"; -const MATERIALIZED_KEY: &str = "materialized"; -const TABLE_MATERIALIZATION: &str = "table"; - -/// `(name, type)`. -pub(crate) type NamedType = (String, Option); -positional_record! { - extract; - /// `(output proven, bare dynamic pivot, (family key, inferred type), failure, (family, type))`. - pub(crate) struct ProofPy { - pub(crate) output_proven: bool, - pub(crate) bare_dynamic_pivot: bool, - pub(crate) families: Vec, - pub(crate) failure_reason: Option, - pub(crate) named_families: Vec, - } -} - -positional_record! { - extract; - /// `(inferred, (name, type), nullable per column, has star, unchecked output columns)`. - pub(crate) struct AnalysisPy { - pub(crate) inferred: bool, - pub(crate) inferred_columns: Vec, - pub(crate) nullable_columns: Vec, - pub(crate) fast_lineage_has_star: bool, - pub(crate) unchecked: Vec, - } -} - -positional_record! { - extract; - /// A model's name, relative path, compiled and authored queries, and authored SQL file. - pub(crate) struct ModelTextsPy { - name: String, - relative_path: String, - query_sql: String, - authored_query_sql: String, - authored_sql: String, - } -} - -/// Encoded config values and the encoder's error, if any. -pub(crate) type ConfigValues = (BTreeMap, Option); - -/// One model as the compile-input stage produced it. -#[derive(FromPyObject)] -pub(crate) struct ModelInputPy<'py>( - ModelTextsPy, - (Bound<'py, PyDict>, Vec, Option), - Vec<(String, String, Option)>, - Vec, - (Vec>, Vec>), -); - -/// Name, relative path, members, value, value type and rendering of one declaration. -#[derive(FromPyObject)] -pub(crate) struct DeclarationPy<'py>( - String, - String, - Vec<(String, Bound<'py, PyAny>)>, - Bound<'py, PyAny>, - Option, - Option, -); - -/// One compile's project, recorded by the stages that produce it, with its Python projections. -#[pyclass(module = "sqlbuild._native")] -#[derive(Default)] -pub(crate) struct NativeCompiledProject { - project: CompiledProject, - /// The Python classes model input views project their attributes with. - input_classes: Option>, - /// Shared immutable column projections, allocated only when Python consumers request them. - inferred_column_projections: Mutex>>, - /// The model loop and its constant rendering, which attach the project's other resources. - attachment: Option<(Py, Py)>, -} - -impl NativeCompiledProject { - /// The facts recorded so far, for native consumers such as the rules request. - pub(crate) fn facts(&self) -> &CompiledProjectFacts { - &self.project.facts - } - - /// A project holding only `facts`, which analyses can be recorded into apart from the compile's. - pub(crate) fn with_facts(facts: CompiledProjectFacts) -> Self { - Self { - project: CompiledProject::with_facts(facts), - ..Self::default() - } - } - - /// Retain facts a producer finished natively. - pub(crate) fn retain(&mut self, facts: CompiledModelFacts) { - self.project.retain(facts); - } - - /// Retain a finished model's input record; return its index for the model's view. - pub(crate) fn retain_input(&mut self, record: ModelInputRecord) -> usize { - self.project.retain_input(record) - } - - /// Keep the classes model input views project their attributes with. - pub(crate) fn set_input_classes(&mut self, classes: Py) { - self.input_classes = Some(classes); - } - - /// Count one lineage graph a project view built. - pub(crate) fn record_graph_build(&mut self) { - self.project.graph_builds += 1; - } - - /// The kept model loop and its constant rendering. - pub(crate) fn attachment_loop( - &self, - py: Python<'_>, - ) -> PyResult<(Py, Py)> { - match &self.attachment { - Some((model_loop, host)) => Ok((model_loop.clone_ref(py), host.clone_ref(py))), - None => Err(PyRuntimeError::new_err(ATTACHMENT_LOOP_UNSET)), - } - } - - /// Keep the model loop and its constant rendering for attaching the project's other resources. - pub(crate) fn keep_attachment_loop( - &mut self, - model_loop: Py, - host: Py, - ) { - self.attachment = Some((model_loop, host)); - } - - /// Retain attached functions; return the index of the first. - pub(crate) fn extend_functions(&mut self, attached: Vec) -> usize { - let first: usize = self.project.functions.len(); - self.project - .functions - .extend(attached.into_iter().map(function_record)); - first - } - - /// Retain attached audits; return the index of the first. - pub(crate) fn extend_audits(&mut self, attached: Vec) -> usize { - let first: usize = self.project.audits.len(); - self.project - .audits - .extend(attached.into_iter().map(audit_record)); - first - } - - /// Every attached audit, in attachment order. - pub(crate) fn audits(&self) -> &[AuditRecord] { - self.project.audits() - } - - /// Every attached function, in attachment order. - pub(crate) fn functions(&self) -> &[FunctionRecord] { - self.project.functions() - } - - /// Every attached source, in attachment order. - pub(crate) fn sources(&self) -> &[SourceRecord] { - self.project.sources() - } - - /// Keep one attached source. - pub(crate) fn retain_source(&mut self, source: SourceRecord) { - self.project.sources.push(source); - } - - /// Every attached seed, in attachment order. - pub(crate) fn seeds(&self) -> &[SeedRecord] { - self.project.seeds() - } - - /// The schema fact recorded for an input record's model. - pub(crate) fn input_schema_fact(&self, record: &ModelInputRecord) -> Option<&ModelSchemaFact> { - self.project.input_schema_fact(record) - } - - /// Every model's input record, in loop order. - pub(crate) fn inputs(&self) -> &[ModelInputRecord] { - &self.project.inputs - } - - /// The input record at `index`. - pub(crate) fn input(&self, index: usize) -> PyResult<&ModelInputRecord> { - self.project - .input(index) - .ok_or_else(|| PyIndexError::new_err(index)) - } - - /// The input record at `index`, for recording what later stages finish. - pub(crate) fn input_mut(&mut self, index: usize) -> PyResult<&mut ModelInputRecord> { - self.project - .inputs - .get_mut(index) - .ok_or_else(|| PyIndexError::new_err(index)) - } - - /// Record that the native parser skipped the model at `index`: it is not analysed. - pub(crate) fn mark_analysis_skipped(&mut self, index: usize) -> PyResult<()> { - let record: &mut ModelInputRecord = self.input_mut(index)?; - record.sql_validation_enabled = false; - let name: String = record.name.clone(); - record_skipped_analysis(&mut self.project.facts, &name); - Ok(()) - } - - /// The classes model input views project their attributes with. - pub(crate) fn input_classes<'py>(&self, py: Python<'py>) -> PyResult> { - self.input_classes - .as_ref() - .map(|classes| classes.bind(py).clone()) - .ok_or_else(|| PyIndexError::new_err(INPUT_CLASSES_UNSET)) - } - - pub(crate) fn inferred_column<'py>( - &self, - py: Python<'py>, - classes: &CompiledViewClasses<'py>, - row: &InferredRow, - ) -> PyResult> { - let (name, column_type, nullability) = row; - if let Some(column) = self - .inferred_column_projections - .lock() - .map_err(|error| native_compiler_error(error.to_string()))? - .get(row) - { - return Ok(column.bind(py).clone()); - } - let fields = PyDict::new(py); - fields.set_item("name", name)?; - fields.set_item("type", column_type.as_deref())?; - fields.set_item("nullability", classes.nullability.call1((nullability,))?)?; - let column = classes.inferred_column.call((), Some(&fields))?; - self.inferred_column_projections - .lock() - .map_err(|error| native_compiler_error(error.to_string()))? - .insert(row.clone(), column.clone().unbind()); - Ok(column) - } -} - -#[pymethods] -impl NativeCompiledProject { - #[new] - fn new() -> Self { - Self::default() - } - - /// Attach a recorded model's typed analysis; `False` when that model was never recorded. - pub(crate) fn record_model_analysis( - &mut self, - name: &str, - columns: AnalysisPy, - dynamic_proof: Option, - ) -> bool { - let AnalysisPy { - inferred, - inferred_columns, - nullable_columns, - fast_lineage_has_star, - unchecked, - } = columns; - let analysis: ModelAnalysisFact = ModelAnalysisFact { - inferred_columns, - dynamic_proof: dynamic_proof.map( - |ProofPy { - output_proven, - bare_dynamic_pivot, - families, - failure_reason, - named_families, - }| DynamicProofFact { - output_proven, - bare_dynamic_pivot, - families, - failure_reason, - named_families, - }, - ), - inferred, - nullable_columns, - fast_lineage_has_star, - unchecked_output_columns: unchecked, - }; - record_model_analysis(&mut self.project.facts, name, analysis) - } - - /// Record each assembled native model view's final typed analysis; return how many. - pub(crate) fn record_view_analyses( - slf: &Bound<'_, Self>, - views: Vec>, - ) -> PyResult { - let py: Python<'_> = slf.py(); - let mut facts: Vec<(String, ModelAnalysisFact)> = Vec::with_capacity(views.len()); - for view in &views { - facts.push(view.analysis_fact(py)?); - } - let mut owner = slf.borrow_mut(); - let mut recorded: usize = 0; - for (name, analysis) in facts { - recorded += usize::from(record_model_analysis( - &mut owner.project.facts, - &name, - analysis, - )); - } - Ok(recorded) - } - - /// The compile inputs view of this owner's model inputs with these input fields. - pub(crate) fn inputs_view( - slf: Bound<'_, Self>, - fields: Bound<'_, PyDict>, - ) -> PyResult { - NativeInputsView::new(slf.unbind(), &fields) - } - - /// The compiled project view of this owner's models with these project fields. - pub(crate) fn project_view( - slf: Bound<'_, Self>, - fields: Bound<'_, PyDict>, - ) -> PyResult { - NativeProjectView::new(slf.unbind(), &fields) - } - - /// Pair seed declarations with their CSV files and keep them; return each seed's view. - pub(crate) fn attach_seeds( - slf: &Bound<'_, Self>, - declarations: Vec<(Py, Py)>, - seed_files: Vec>, - compile_error: &Bound<'_, PyAny>, - ) -> PyResult> { - let seeds: Vec = paired_seeds(declarations, &seed_files, compile_error)?; - let count: usize = seeds.len(); - slf.borrow_mut().project.seeds = seeds; - Ok((0..count) - .map(|index| NativeSeedInput::new(slf.clone().unbind(), index)) - .collect()) - } - - /// Keep the destinations native project assembly resolved, in seed order. - fn record_seed_namespaces(&mut self, namespaces: Vec) -> PyResult<()> { - for (index, namespace) in namespaces.into_iter().enumerate() { - self.project - .seeds - .get_mut(index) - .ok_or_else(|| PyIndexError::new_err(index))? - .set_namespace(namespace); - } - Ok(()) - } - - /// Attach every discovered source in order with the kept loop; return each source's view. - pub(crate) fn attach_sources( - slf: &Bound<'_, Self>, - rows: Vec, - settings: SourceSettings, - ) -> PyResult> { - (resource_attachment()?.sources)(slf, rows, settings) - } - - /// Attach every SQL, then Python, function with the kept loop; return each one's view. - pub(crate) fn attach_functions( - slf: &Bound<'_, Self>, - rows: (Vec, Vec), - settings: FunctionSettings, - ) -> PyResult> { - (resource_attachment()?.functions)(slf, rows, settings) - } - - /// The orphaned audit factory warnings, missing descriptions and cursor-input errors. - fn resource_diagnostics<'py>( - &self, - py: Python<'py>, - settings: DiagnosticSettings, - ) -> PyResult> { - (resource_attachment()?.diagnostics)(self, py, &settings) - } - - /// Attach every SQL test case, then every scenario, with the kept loop. - pub(crate) fn attach_sql_tests( - slf: &Bound<'_, Self>, - files: Vec, - scenarios: Vec, - settings: SqlTestSettings, - ) -> PyResult { - (resource_attachment()?.sql_tests)(slf, files, scenarios, settings) - } - - /// Attach singular, then resource-attached, audits; return views and override warnings. - pub(crate) fn attach_audits<'py>( - slf: &Bound<'py, Self>, - files: Vec, - settings: AuditSettings, - ) -> PyResult<(Vec, Vec>)> { - (resource_attachment()?.audits)(slf, files, settings) - } - /// Keep the scope dependency keys native project assembly resolved, in audit order. - fn record_audit_scope_deps(&mut self, deps: Vec>) -> PyResult<()> { - for (index, keys) in deps.into_iter().enumerate() { - self.project - .audits - .get_mut(index) - .ok_or_else(|| PyIndexError::new_err(index))? - .set_scope_deps(keys); - } - Ok(()) - } - - /// The assembled audit at `index`. - fn compiled_audit(slf: &Bound<'_, Self>, index: usize) -> PyResult { - if index >= slf.borrow().project.audits.len() { - return Err(PyIndexError::new_err(index)); - } - Ok(NativeCompiledAudit::new(slf.clone().unbind(), index)) - } - - /// Keep the dependency keys native project assembly resolved, in function order. - fn record_function_deps(&mut self, deps: Vec>) -> PyResult<()> { - for (index, keys) in deps.into_iter().enumerate() { - self.project - .functions - .get_mut(index) - .ok_or_else(|| PyIndexError::new_err(index))? - .set_deps(keys); - } - Ok(()) - } - - /// The assembled function at `index`. - fn compiled_function(slf: &Bound<'_, Self>, index: usize) -> PyResult { - if index >= slf.borrow().project.functions.len() { - return Err(PyIndexError::new_err(index)); - } - Ok(NativeCompiledFunction::new(slf.clone().unbind(), index)) - } - - /// Keep the namespaces native project assembly resolved, in source order. - fn record_source_namespaces(&mut self, namespaces: Vec) -> PyResult<()> { - for (index, namespace) in namespaces.into_iter().enumerate() { - self.project - .sources - .get_mut(index) - .ok_or_else(|| PyIndexError::new_err(index))? - .set_namespace(namespace); - } - Ok(()) - } - - /// The assembled source at `index`. - fn compiled_source(slf: &Bound<'_, Self>, index: usize) -> PyResult { - if index >= slf.borrow().project.sources.len() { - return Err(PyIndexError::new_err(index)); - } - Ok(NativeCompiledSource::new(slf.clone().unbind(), index)) - } - - /// The assembled seed at `index`. - fn compiled_seed(slf: &Bound<'_, Self>, index: usize) -> PyResult { - if index >= slf.borrow().project.seeds.len() { - return Err(PyIndexError::new_err(index)); - } - Ok(NativeCompiledSeed::new(slf.clone().unbind(), index)) - } - - /// How many lineage graphs this owner's project views built. - fn graph_builds(&self) -> usize { - self.project.graph_builds - } - - /// Whether both projects retained the same facts. - fn __eq__(&self, other: &Self) -> bool { - self.project.facts == other.project.facts - } - - /// The recorded models' names, in production order. - fn model_names(&self) -> Vec { - project_models(&self.project.facts) - .iter() - .map(|model| model.name.clone()) - .collect() - } -} - -pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult { - let ModelInputPy( - ModelTextsPy { - name, - relative_path, - query_sql, - authored_query_sql, - authored_sql, - }, - (values, header_keys, layer_schema), - references, - enum_columns, - (enum_declarations, constant_declarations), - ) = model; - let config_values: ConfigValues = config_values(&values); - Ok(CompiledModelFacts { - name, - relative_path, - query_sql: query_sql.into(), - authored_query_sql: authored_query_sql.into(), - authored_sql: authored_sql.into(), - config: ModelConfigFacts { - values: config_values.0, - encoding_error: config_values.1, - header_keys, - layer_schema, - contract: text_value(&values, CONTRACT_KEY)?, - materialized: materialization(&values)?, - incremental_mode: text_value(&values, INCREMENTAL_MODE_KEY)?, - }, - references: references - .into_iter() - .map(|(kind, name, package)| ModelReferenceFact { - kind, - name, - package, - }) - .collect(), - schema: None, - enum_columns, - enum_declarations: declarations(enum_declarations)?, - constant_declarations: declarations(constant_declarations)?, - analysis: None, - typed_calls: Vec::new(), - expansion_passes: Vec::new(), - sql_analysis_skipped: false, - }) -} - -/// The config values, or the encoder's error, kept for the consumer that reads them. -pub(crate) fn config_values(values: &Bound<'_, PyDict>) -> ConfigValues { - match object_value(values) { - Ok(Value::Object(entries)) => (entries.into_iter().collect(), None), - Ok(_) => (BTreeMap::new(), None), - Err(error) => (BTreeMap::new(), Some(error.to_string())), - } -} - -/// The `materialized` config value when it is text, `table` when it is absent. -pub(crate) fn materialization(values: &Bound<'_, PyDict>) -> PyResult> { - if values.get_item(MATERIALIZED_KEY)?.is_none() { - return Ok(Some(TABLE_MATERIALIZATION.to_owned())); - } - text_value(values, MATERIALIZED_KEY) -} - -/// A config value when it is text. -pub(crate) fn text_value(values: &Bound<'_, PyDict>, key: &str) -> PyResult> { - match values.get_item(key)? { - Some(value) if value.is_instance_of::() => Ok(Some(value.extract()?)), - _ => Ok(None), - } -} - -pub(crate) fn declarations(rows: Vec>) -> PyResult> { - rows.into_iter().map(declaration).collect() -} - -fn declaration(row: DeclarationPy<'_>) -> PyResult { - let DeclarationPy(name, relative_path, members, value, value_type, render_as) = row; - Ok(DeclarationFact { - name, - relative_path, - members: members - .into_iter() - .map(|(name, value)| Ok((name, plain_value(&value)?))) - .collect::>()?, - value: Some(plain_value(&value)?).filter(|value| !value.is_null()), - value_type, - render_as, - }) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs index 2c8daa3464..2c8f452329 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs @@ -1,12 +1,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::{compiled_views, input_loops}; +use sqlbuild_python_project::compiled_project::main::register_project::register_project; +use sqlbuild_python_project::project_snapshot::main::register_snapshot::register_snapshot; /// Register the compiled project, then its input loops, then its views and records. pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - crate::bindings::_helpers::compiled_project::project::register(module)?; + register_project(module)?; input_loops::registration::register(module)?; compiled_views::registration::register(module)?; crate::bindings::_helpers::location_command::command::register(module)?; - crate::bindings::_helpers::compiled_project::snapshot::register(module) + register_snapshot(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/rows.rs deleted file mode 100644 index 5499367ebd..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/rows.rs +++ /dev/null @@ -1,31 +0,0 @@ -//! Positional rows shared by the compiled views and the input loops that fill them. - -use sqlbuild_project::models::ReferenceRow; -use sqlbuild_python_base::positional_record; - -positional_record! { - into; - /// A reference as the tuple Python reads: kind, name, package and call argument count. - pub(crate) struct ReferenceTuple { - pub(crate) kind: String, - pub(crate) name: String, - package: Option, - call_argument_count: Option, - } -} - -/// A reference as the tuple Python reads. -pub(crate) fn reference_row_tuple(row: ReferenceRow) -> ReferenceTuple { - let ReferenceRow { - kind, - name, - package, - call_argument_count, - } = row; - ReferenceTuple { - kind, - name, - package, - call_argument_count, - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs deleted file mode 100644 index d22d801145..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs +++ /dev/null @@ -1,134 +0,0 @@ -//! Native facts of declared schema columns and families, read once from their Python specs. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_analysis::compiled_project::models::{ - DynamicFamilyFact, ModelSchemaFact, SchemaColumnFact, -}; -use sqlbuild_core::text::main::python_casefold::python_casefold; -use sqlbuild_model_config::header_metadata::models::DynamicFamily; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::record_attribute_opt::record_attribute_opt; - -const NAME_FIELD: &str = "name"; -const TYPE_FIELD: &str = "type"; -const NULLABLE_FIELD: &str = "nullable"; -const DESCRIPTION_FIELD: &str = "description"; -const MIGRATE_FROM_FIELD: &str = "migrate_from"; -const AUDITS_FIELD: &str = "audits"; -const LOCATION_FIELD: &str = "location"; -const PATH_FIELD: &str = "path"; -const DEFINITION_NAME_FIELD: &str = "definition_name"; -const COLUMNS_FIELD: &str = "columns"; -const DYNAMIC_COLUMNS_FIELD: &str = "dynamic_columns"; -const TYPE_ENFORCEMENT_FIELD: &str = "type_enforcement"; -const MODEL_SCHEMA_FIELD: &str = "model_schema"; -const PIVOT_COLUMN_FIELD: &str = "pivot_column"; -const VALUE_COLUMN_FIELD: &str = "value_column"; -const AGGREGATE_FIELD: &str = "aggregate"; -const NAME_PATTERN_FIELD: &str = "name_pattern"; - -/// A Python column spec's fact; its location is kept when it differs from `model_path`. -pub(crate) fn python_column_fact( - column: &Bound<'_, PyAny>, - model_path: Option<&Bound<'_, PyAny>>, -) -> PyResult { - let mut audit_definitions: Vec = Vec::new(); - if let Some(audits) = present(column, AUDITS_FIELD)? { - for audit in audits.try_iter()? { - audit_definitions.push(native_field(&audit?, DEFINITION_NAME_FIELD)?.extract()?); - } - } - let location_path: Option = match present(column, LOCATION_FIELD)? { - Some(location) => { - let path: Bound<'_, PyAny> = native_field(&location, PATH_FIELD)?; - let outside: bool = match model_path { - Some(model_path) => !path.is(model_path) && path.ne(model_path)?, - None => true, - }; - outside - .then(|| path.str().map(|text| text.to_string())) - .transpose()? - } - None => None, - }; - Ok(SchemaColumnFact { - name: native_field(column, NAME_FIELD)?.str()?.extract()?, - data_type: optional(column, TYPE_FIELD)?, - nullable: optional(column, NULLABLE_FIELD)?, - described: present(column, DESCRIPTION_FIELD)?.is_some(), - migrate_from: optional(column, MIGRATE_FROM_FIELD)?, - audit_definitions, - location_path, - }) -} - -/// The facts of every column spec in `columns`. -pub(crate) fn python_column_facts( - columns: &Bound<'_, PyAny>, - model_path: Option<&Bound<'_, PyAny>>, -) -> PyResult> { - columns - .try_iter()? - .map(|column| python_column_fact(&column?, model_path)) - .collect() -} - -/// The fact of a hand-built model's `SchemaModelEntry`, or `None` without one. -pub(crate) fn entry_schema_fact( - entry: &Bound<'_, PyAny>, - model_path: &Bound<'_, PyAny>, -) -> PyResult> { - if entry.is_none() { - return Ok(None); - } - let mut dynamic_columns: Vec = Vec::new(); - for family in native_field(entry, DYNAMIC_COLUMNS_FIELD)?.try_iter()? { - let family: Bound<'_, PyAny> = family?; - dynamic_columns.push(family_fact(&DynamicFamily { - name: native_field(&family, NAME_FIELD)?.extract()?, - pivot_column: native_field(&family, PIVOT_COLUMN_FIELD)?.extract()?, - value_column: native_field(&family, VALUE_COLUMN_FIELD)?.extract()?, - aggregate: native_field(&family, AGGREGATE_FIELD)?.extract()?, - data_type: native_field(&family, TYPE_FIELD)?.extract()?, - name_pattern: native_field(&family, NAME_PATTERN_FIELD)?.extract()?, - })); - } - Ok(Some(ModelSchemaFact { - audit_count: u32::try_from(native_field(entry, AUDITS_FIELD)?.len()?).unwrap_or(u32::MAX), - columns: python_column_facts(&native_field(entry, COLUMNS_FIELD)?, Some(model_path))?, - dynamic_columns, - type_enforcement: native_field(entry, TYPE_ENFORCEMENT_FIELD)?.is_truthy()?, - named_schema: !native_field(entry, MODEL_SCHEMA_FIELD)?.is_none(), - })) -} - -/// A parsed dynamic column family's fact, keyed by its casefolded name. -pub(crate) fn family_fact(family: &DynamicFamily) -> DynamicFamilyFact { - DynamicFamilyFact { - key: python_casefold(&family.name), - name: family.name.clone(), - pivot_column: family.pivot_column.clone(), - value_column: family.value_column.clone(), - aggregate: family.aggregate.clone(), - data_type: family.data_type.clone(), - name_pattern: family.name_pattern.clone(), - } -} - -/// An attribute's value, `None` when it is missing or `None`. -fn present<'py>(spec: &Bound<'py, PyAny>, name: &str) -> PyResult>> { - let value: Option> = match record_attribute_opt(spec, name) { - Some(value) => value?, - None => spec.getattr_opt(name)?, - }; - Ok(value.filter(|value| !value.is_none())) -} - -fn optional<'py, T: pyo3::FromPyObject<'py>>( - spec: &Bound<'py, PyAny>, - name: &str, -) -> PyResult> { - present(spec, name)? - .map(|value| value.extract()) - .transpose() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs deleted file mode 100644 index 089dfedbfc..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot.rs +++ /dev/null @@ -1,107 +0,0 @@ -//! Project metadata stays native until its immutable snapshot is sent over HTTP. - -use std::collections::BTreeMap; - -use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyclass, pymethods}; -use serde_json::Value; -use sqlbuild_orchestration::api::errors::ClientError; -use sqlbuild_orchestration::snapshots::main::publish_remote_snapshot::publish_remote_snapshot; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode, SnapshotRecord}; - -use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; - -/// A data-only publication; this object never contains adapters, connections or user callables. -#[pyclass(frozen, module = "sqlbuild._native")] -pub(crate) struct NativeOrchestrationSnapshot { - pub(crate) input: SnapshotInput, -} - -impl NativeOrchestrationSnapshot { - /// Native callers retain transport errors so they can report an uncertain acknowledgement. - pub(crate) fn publication( - &self, - py: Python<'_>, - server_url: &str, - bearer: &str, - ) -> PyResult> { - py.detach(|| { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; - let result = runtime.block_on(publish_remote_snapshot(server_url, bearer, &self.input)); - runtime.shutdown_timeout(std::time::Duration::from_secs(1)); - Ok(result) - }) - } -} - -#[pymethods] -impl NativeOrchestrationSnapshot { - /// Serialize only at the public boundary; native transports can use the retained input. - fn to_json(&self, py: Python<'_>) -> PyResult { - py.detach(|| serde_json::to_string(&self.input)) - .map_err(|error| PyValueError::new_err(error.to_string())) - } - - /// Publish retained data without projecting it through Python, releasing the GIL during IO. - fn publish(&self, py: Python<'_>, server_url: &str, bearer: &str) -> PyResult { - let record = self - .publication(py, server_url, bearer)? - .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; - py.detach(|| { - serde_json::to_string(&record) - .map_err(|error| PyRuntimeError::new_err(error.to_string())) - }) - } - - /// Retain the verified claim and its independent native heartbeat in the worker interpreter. - fn start_worker( - &self, - py: Python<'_>, - server_url: &str, - bearer: &str, - identity: (String, String), - ) -> PyResult { - NativeOrchestrationWorker::start(self, py, (server_url, bearer), identity) - } -} - -/// Duplicate resource identities must fail, rather than silently replacing an earlier node. -pub(crate) fn insert_node( - mut nodes: BTreeMap, - key: String, - node: SnapshotNode, -) -> PyResult> { - if nodes.contains_key(&key) { - return Err(PyValueError::new_err(format!( - "duplicate snapshot node identity {key}" - ))); - } - nodes.insert(key, node); - Ok(nodes) -} - -pub(crate) fn dependency_keys(deps: &[(String, String)]) -> Vec { - deps.iter() - .map(|(kind, name)| format!("{kind}:{name}")) - .collect() -} - -pub(crate) fn destination(value: &sqlbuild_project::models::DestinationFact) -> Value { - serde_json::json!({ - "database": value.database, - "schema": value.schema, - "name": value.name, - "qualified_name": value.qualified_name, - "logical_database": value.logical_database, - "logical_schema": value.logical_schema, - }) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs deleted file mode 100644 index aa2342912c..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_worker.rs +++ /dev/null @@ -1,217 +0,0 @@ -//! A worker interpreter retains the native claim owner; it never opens the control database. - -use std::sync::Mutex; - -use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{PyRef, PyResult, Python}; -use pyo3::{pyclass, pymethods}; -use sqlbuild_orchestration::runs::models::RunChange; -use sqlbuild_orchestration::worker::errors::WorkerError; -use sqlbuild_orchestration::worker::main::connect_worker::connect_worker; -use sqlbuild_orchestration::worker::main::start_worker::start_worker; -use sqlbuild_orchestration::worker::models::{WorkerClient, WorkerSession}; -use uuid::Uuid; - -use crate::bindings::_helpers::compiled_project::constants::{REPORT_ATTEMPTS, STORE_BUSY_STATUS}; -use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; - -struct WorkerOwner { - runtime: tokio::runtime::Runtime, - client: WorkerClient, - session: WorkerSession, -} - -/// Closing this owner stops and joins heartbeat IO but does not claim warehouse work has stopped. -#[pyclass(frozen, module = "sqlbuild._native")] -pub(crate) struct NativeOrchestrationWorker { - owner: Mutex>, -} - -impl NativeOrchestrationWorker { - /// Run `work` on the open owner without holding the GIL; a closed worker is unavailable. - fn with_owner( - &self, - py: Python<'_>, - work: impl FnOnce(&WorkerOwner) -> PyResult + Send, - ) -> PyResult { - py.detach(|| { - let owner = self.owner.lock().map_err(|_| unavailable())?; - work(owner.as_ref().ok_or_else(unavailable)?) - }) - } - - /// SIGTERM may arrive before the periodic heartbeat observes the durable cancellation fact. - pub(crate) fn refresh_cancellation(&self, py: Python<'_>) -> PyResult { - self.with_owner(py, |owner| { - if owner.session.cancel_requested() { - return Ok(true); - } - owner - .runtime - .block_on(owner.client.heartbeat()) - .map_err(|error| PyRuntimeError::new_err(error.to_string())) - }) - } - - /// Native execution reads the retained request, never caller-provided selection or settings. - pub(crate) fn execution( - &self, - py: Python<'_>, - ) -> PyResult { - self.with_owner(py, |owner| Ok(owner.session.execution.clone())) - } - - /// A bounded retry keeps the event identity across an uncertain acknowledgement or busy store. - pub(crate) fn change(&self, py: Python<'_>, key: &str, change: RunChange) -> PyResult<()> { - self.with_owner(py, |owner| { - let event_id = Uuid::new_v5(&owner.session.execution.claim.claim_id, key.as_bytes()); - owner.runtime.block_on(async { - let mut attempts = 0; - loop { - attempts += 1; - match owner.client.report(event_id, change.clone()).await { - Ok(_) => return Ok(()), - Err(error) - if attempts < REPORT_ATTEMPTS - && match &error { - WorkerError::Transport(_) => true, - WorkerError::Rejected(status) => { - status.as_u16() == STORE_BUSY_STATUS - } - _ => false, - } => - { - tokio::time::sleep(std::time::Duration::from_millis(200)).await; - } - Err(error) => return Err(PyRuntimeError::new_err(error.to_string())), - } - } - }) - }) - } - - /// Verify this retained baseline before the caller applies run settings or opens a warehouse. - pub(crate) fn start( - snapshot: &NativeOrchestrationSnapshot, - py: Python<'_>, - (server_url, bearer): (&str, &str), - (run_id, claim_id): (String, String), - ) -> PyResult { - let identity = ( - Uuid::parse_str(&run_id).map_err(|error| PyValueError::new_err(error.to_string()))?, - Uuid::parse_str(&claim_id).map_err(|error| PyValueError::new_err(error.to_string()))?, - ); - py.detach(|| { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; - let connected = runtime.block_on(async { - let client = connect_worker(server_url, bearer, identity)?; - let session = start_worker(&client, &snapshot.input).await?; - Ok::<_, sqlbuild_orchestration::worker::errors::WorkerError>((client, session)) - }); - match connected { - Ok((client, session)) => Ok(NativeOrchestrationWorker { - owner: Mutex::new(Some(WorkerOwner { - runtime, - client, - session, - })), - }), - Err(error) => { - runtime.shutdown_timeout(std::time::Duration::from_secs(1)); - Err(PyRuntimeError::new_err(error.to_string())) - } - } - }) - } -} - -#[pymethods] -impl NativeOrchestrationWorker { - /// The frozen request supplies selection and settings only after baseline verification. - fn execution_json(&self, py: Python<'_>) -> PyResult { - self.with_owner(py, |owner| { - serde_json::to_string(&owner.session.execution) - .map_err(|error| PyRuntimeError::new_err(error.to_string())) - }) - } - - /// Cancellation remains sticky; a closed worker cannot authorize further work. - pub(crate) fn cancel_requested(&self, py: Python<'_>) -> bool { - py.detach(|| { - self.owner.lock().map_or(true, |owner| { - owner - .as_ref() - .is_none_or(|owner| owner.session.cancel_requested()) - }) - }) - } - - /// Retry an uncertain report with the identical event ID and change, never a new identity. - fn report(&self, py: Python<'_>, event_id: &str, change_json: &str) -> PyResult { - let event_id = - Uuid::parse_str(event_id).map_err(|error| PyValueError::new_err(error.to_string()))?; - let change: RunChange = serde_json::from_str(change_json) - .map_err(|error| PyValueError::new_err(error.to_string()))?; - self.with_owner(py, |owner| { - owner - .runtime - .block_on(owner.client.report(event_id, change)) - .map_err(|error| PyRuntimeError::new_err(error.to_string())) - }) - } - - fn __enter__(slf: PyRef<'_, Self>) -> PyResult> { - if slf.owner.lock().map_err(|_| unavailable())?.is_none() { - return Err(unavailable()); - } - Ok(slf) - } - - fn __exit__( - &self, - py: Python<'_>, - _kind: &pyo3::Bound<'_, pyo3::PyAny>, - _value: &pyo3::Bound<'_, pyo3::PyAny>, - _traceback: &pyo3::Bound<'_, pyo3::PyAny>, - ) -> PyResult<()> { - self.close(py) - } - - /// Join the heartbeat without publishing a terminal outcome or releasing physical claims. - fn close(&self, py: Python<'_>) -> PyResult<()> { - py.detach(|| { - let mut owner = self.owner.lock().map_err(|_| unavailable())?; - if let Some(active) = owner.take() { - active.close(); - } - Ok(()) - }) - } -} - -fn unavailable() -> pyo3::PyErr { - PyRuntimeError::new_err("native worker session is closed or unavailable") -} - -impl WorkerOwner { - fn close(self) { - drop(self.session); - drop(self.client); - self.runtime - .shutdown_timeout(std::time::Duration::from_secs(1)); - } -} - -impl Drop for NativeOrchestrationWorker { - /// GC can hold the GIL; heartbeat teardown joins native IO and never needs Python. - fn drop(&mut self) { - if let Ok(owner) = self.owner.get_mut() - && let Some(owner) = owner.take() - { - owner.close(); - } - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_reads.rs deleted file mode 100644 index 38f2b1c0e3..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_reads.rs +++ /dev/null @@ -1,233 +0,0 @@ -//! A compiled audit's attributes the planner reads, from its native view. - -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; - -use crate::bindings::_helpers::compiled_views::audit_views::NativeCompiledAudit; -use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; -use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -/// The native view behind an audit object; any other object is a Python-built record, counted. -#[inline] -fn native_audit<'a>(audit: &'a Bound<'_, PyAny>) -> PyResult> { - if let Ok(view) = audit.cast::() { - return Ok(Some(view.get())); - } - count_stage(audit.py(), COMPILED_VIEW_FALLBACKS, 1)?; - Ok(None) -} - -/// The compiled audit's `always_run`, as its Python attribute reads it. -pub(crate) fn always_run<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .always_run(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "always_run") -} - -/// The compiled audit's `attached_column_name`, as its Python attribute reads it. -pub(crate) fn attached_column_name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .attached_column_name(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "attached_column_name") -} - -/// The compiled audit's `attached_target_kind`, as its Python attribute reads it. -pub(crate) fn attached_target_kind<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .attached_target_kind(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "attached_target_kind") -} - -/// The compiled audit's `attached_target_name`, as its Python attribute reads it. -pub(crate) fn attached_target_name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .attached_target_name(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "attached_target_name") -} - -/// The compiled audit's `definition_name`, as its Python attribute reads it. -pub(crate) fn definition_name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .definition_name(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "definition_name") -} - -/// The compiled audit's `description`, as its Python attribute reads it. -pub(crate) fn description<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .description(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "description") -} - -/// The compiled audit's `evaluation_mode`, as its Python attribute reads it. -pub(crate) fn evaluation_mode<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .evaluation_mode(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "evaluation_mode") -} - -/// The compiled audit's `evidence_limit`, as its Python attribute reads it. -pub(crate) fn evidence_limit<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .evidence_limit(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "evidence_limit") -} - -/// The compiled audit's `evidence_sql`, as its Python attribute reads it. -pub(crate) fn evidence_sql<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .evidence_sql(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "evidence_sql") -} - -/// The compiled audit's `key`, as its Python attribute reads it. -pub(crate) fn key<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "key") -} - -/// The compiled audit's `measurement_contract`, as its Python attribute reads it. -pub(crate) fn measurement_contract<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .measurement_contract(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "measurement_contract") -} - -/// The compiled audit's `meta`, as its Python attribute reads it. -pub(crate) fn meta<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view.meta(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "meta") -} - -/// The compiled audit's `minimum_samples`, as its Python attribute reads it. -pub(crate) fn minimum_samples<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .minimum_samples(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "minimum_samples") -} - -/// The compiled audit's `name`, as its Python attribute reads it. -pub(crate) fn name<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "name") -} - -/// The compiled audit's `references`, as its Python attribute reads it. -pub(crate) fn references<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .references(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "references") -} - -/// The compiled audit's `run_scope`, as its Python attribute reads it. -pub(crate) fn run_scope<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .run_scope(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "run_scope") -} - -/// The compiled audit's `scope_deps`, as its Python attribute reads it. -pub(crate) fn scope_deps<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .scope_deps(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "scope_deps") -} - -/// The compiled audit's `severity`, as its Python attribute reads it. -pub(crate) fn severity<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .severity(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "severity") -} - -/// The compiled audit's `sql_body`, as its Python attribute reads it. -pub(crate) fn sql_body<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .sql_body(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "sql_body") -} - -/// The compiled audit's `thresholds`, as its Python attribute reads it. -pub(crate) fn thresholds<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = audit.py(); - if let Some(view) = native_audit(audit)? { - return view - .thresholds(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(audit, "thresholds") -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs deleted file mode 100644 index 24e937b063..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs +++ /dev/null @@ -1,942 +0,0 @@ -//! A compiled model, read from the native compiled project that owns it. - -use sqlbuild_project::models as project_models; -use sqlbuild_project::models::{DestinationFact, ModelMetadataTexts}; -use sqlbuild_project::types::InferredRow; -use sqlbuild_python_front::model_config::models::HeaderAudits; -use std::sync::Arc; - -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; -use pyo3::{FromPyObject, pyclass, pymethods}; - -use sqlbuild_analysis::assembly::analysis_session::main::contract_proof_json::contract_proof_json; -use sqlbuild_analysis::assembly::analysis_session::models::LineageRow; -use sqlbuild_analysis::compiled_project::main::model_facts::model_facts; -use sqlbuild_analysis::compiled_project::models::{ - CompiledModelFacts, DynamicProofFact, ModelAnalysisFact, -}; - -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::rows::{ReferenceTuple, reference_row_tuple}; -use crate::bindings::_helpers::compiled_views::compiled_model_analysis::{ - AnalysisRecord, AnalysisViews, ProofArgument, SessionArgument, analysis_record, -}; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - InputViewClasses, ModelInputRecord, model_relation_key_row, query_sql_text, read_input, - read_input_with_owner, references, rejected_opt_out, schema_entry, view_classes, -}; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::compiled_views::relation_keys::RelationKeyRow; -use crate::bindings::_helpers::graph::project_graph::ResourceRow; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::discovery::models::NativeModelFile; -use sqlbuild_python_front::model_config::models::HeaderColumns; - -const DATABASE_KEY: &str = "database"; -const SCHEMA_KEY: &str = "schema"; -const ALIAS_KEY: &str = "alias"; -const VALUES_ATTRIBUTE: &str = "values"; -const LOGICAL_SCHEMA_ATTRIBUTE: &str = "logical_schema"; -const LOGICAL_DATABASE_ATTRIBUTE: &str = "logical_database"; -const MODEL_RESOURCE: &str = "model"; -const SQL_ANALYSIS_KEY: &str = "sql_analysis"; -const TAGS_KEY: &str = "tags"; -const NULLABLE: &str = "nullable"; - -/// A model's SQL rule inputs; `excluded` is `None` when its config values could not be read natively. -pub(crate) struct SqlRuleInputs { - pub(crate) posix_path: String, - pub(crate) excluded: Option, - pub(crate) key: Option, -} - -/// What a model's SQL rule cache key reads. -pub(crate) struct SqlRuleKey { - pub(crate) authored_sql: String, - pub(crate) query_sql: String, - /// The canonical JSON text of the model's dynamic pivot proof. - pub(crate) contract: Option, -} - -/// A schema's entry audits, column specs and each audited column's index and name. -pub(crate) type AuditedSchema = (Arc, Arc, Vec<(usize, String)>); - -/// `(name, type)` of a declared column. -pub(crate) type DeclaredRow = (String, Option); - -/// `(database, schema, name)` of a relation. -pub(crate) type DestinationRow = (Option, Option, String); - -/// `(output column, [(resource name, column name)])` of one lineage output. -pub(crate) type LineageInput = (String, Vec<(String, String)>); - -/// The Python classes a compiled model's projected attributes are built with. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct CompiledViewClasses<'py> { - object_key: Bound<'py, PyAny>, - relation_location: Bound<'py, PyAny>, - pub(crate) inferred_column: Bound<'py, PyAny>, - pub(crate) nullability: Bound<'py, PyAny>, - pub(crate) compiler_diagnostic: Bound<'py, PyAny>, - pub(crate) source_location: Bound<'py, PyAny>, - pub(crate) compact_lineage_facts: Bound<'py, PyAny>, - pub(crate) lineage_column_fact: Bound<'py, PyAny>, - pub(crate) lineage_source_fact: Bound<'py, PyAny>, - pub(crate) contract_proof: Bound<'py, PyAny>, - pub(crate) family_proof: Bound<'py, PyAny>, -} - -/// What assembly and the semantic checks finished for one compiled model. -pub(crate) type CompiledRecord = project_models::CompiledRecord; - -/// A compiled model's Python attribute projections, each built once on first access. -pub(crate) struct CompiledProjections { - key: PyOnceLock>, - deps: PyOnceLock>, -} - -impl Default for CompiledProjections { - fn default() -> Self { - Self { - key: PyOnceLock::new(), - deps: PyOnceLock::new(), - } - } -} - -/// What later stages changed for one view of a model; the owner's record stays as assembled. -#[derive(Default)] -struct ModelOverlay { - binding_diagnostics: Option>, - /// Output columns whose types semantic recovery forgot, left unchecked by the contract. - poisoned: Vec, - destination: Option, -} - -impl ModelOverlay { - fn clone_ref(&self, py: Python<'_>) -> Self { - Self { - binding_diagnostics: self - .binding_diagnostics - .as_ref() - .map(|diagnostics| diagnostics.clone_ref(py)), - poisoned: self.poisoned.clone(), - destination: self.destination.clone(), - } - } -} - -/// One view's projections of attributes its overlay can change, each built once. -struct ViewProjections { - destination: PyOnceLock>, - inferred_columns: PyOnceLock>, -} - -impl Default for ViewProjections { - fn default() -> Self { - Self { - destination: PyOnceLock::new(), - inferred_columns: PyOnceLock::new(), - } - } -} - -/// Deps, session analysis, failed analysis's columns, standalone proof, and macro deps. -#[derive(FromPyObject)] -pub(crate) struct CompiledArguments( - Vec<(String, String)>, - Option, - Option>, - Option, - Vec, -); - -/// The compiled record of one model, with its configured destination. -pub(crate) fn compiled_record( - py: Python<'_>, - input: &ModelInputRecord, - name: &str, - arguments: CompiledArguments, -) -> PyResult { - let CompiledArguments(deps, session, fallback, standalone_proof, macro_deps) = arguments; - let analysis: AnalysisRecord = - analysis_record(py, input, session, (fallback, standalone_proof))?; - let config: &Bound<'_, PyAny> = input.host.config.bind(py); - let values: Bound<'_, PyDict> = native_field(config, VALUES_ATTRIBUTE)?.downcast_into()?; - Ok(CompiledRecord { - deps, - destination: DestinationFact { - database: text_value(&values, DATABASE_KEY)?, - schema: text_value(&values, SCHEMA_KEY)?, - name: text_value(&values, ALIAS_KEY)?.unwrap_or_else(|| name.to_owned()), - qualified_name: None, - logical_schema: native_field(config, LOGICAL_SCHEMA_ATTRIBUTE)?.extract()?, - logical_database: native_field(config, LOGICAL_DATABASE_ATTRIBUTE)?.extract()?, - }, - analysis, - macro_deps, - host: CompiledProjections::default(), - }) -} - -fn text_value(values: &Bound<'_, PyDict>, key: &str) -> PyResult> { - match values.get_item(key)? { - Some(value) if value.is_instance_of::() => { - Ok(Some(value.extract()?)) - } - _ => Ok(None), - } -} - -/// The destination with adapter defaults filled in; `render` renders its qualified name. -pub(crate) fn defaulted_destination( - current: &DestinationFact, - defaults: (Option, Option), - render: &Bound<'_, PyAny>, -) -> PyResult { - let (default_schema, default_database) = defaults; - let schema: Option = current.schema.clone().or(default_schema); - let database: Option = current.database.clone().or(default_database); - let keywords: Bound<'_, PyDict> = PyDict::new(render.py()); - keywords.set_item(DATABASE_KEY, &database)?; - keywords.set_item(SCHEMA_KEY, &schema)?; - keywords.set_item("name", ¤t.name)?; - let qualified_name: Option = render.call((), Some(&keywords))?.extract()?; - Ok(DestinationFact { - database, - schema, - name: current.name.clone(), - qualified_name, - logical_schema: current.logical_schema.clone(), - logical_database: current.logical_database.clone(), - }) -} - -/// One compiled model, projected from the owner on demand. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeCompiledModel { - owner: Py, - index: usize, - overlay: ModelOverlay, - projections: ViewProjections, -} - -impl NativeCompiledModel { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self::with_overlay(owner, index, ModelOverlay::default()) - } - - fn with_overlay(owner: Py, index: usize, overlay: ModelOverlay) -> Self { - Self { - owner, - index, - overlay, - projections: ViewProjections::default(), - } - } - - fn changed(&self, py: Python<'_>, change: impl FnOnce(&mut ModelOverlay)) -> Self { - let mut overlay: ModelOverlay = self.overlay.clone_ref(py); - change(&mut overlay); - Self::with_overlay(self.owner.clone_ref(py), self.index, overlay) - } - - pub(crate) fn read<'py, T>( - &self, - py: Python<'py>, - read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, - ) -> PyResult { - read_input(&self.owner, self.index, py, read) - } - - fn compiled<'py, T>( - &self, - py: Python<'py>, - read: impl FnOnce(&CompiledRecord, Python<'py>) -> PyResult, - ) -> PyResult { - self.read(py, |record, py| read(compiled_of(record)?, py)) - } - - /// Read the effective destination, including namespace overlays, without a Python projection. - pub(crate) fn read_destination( - &self, - py: Python<'_>, - read: impl FnOnce(&DestinationFact) -> PyResult, - ) -> PyResult { - self.compiled(py, |compiled, _| { - read( - self.overlay - .destination - .as_ref() - .unwrap_or(&compiled.destination), - ) - }) - } - - /// This view's typed analysis as the owner's model facts read it, and the model's name. - pub(crate) fn analysis_fact(&self, py: Python<'_>) -> PyResult<(String, ModelAnalysisFact)> { - self.read(py, |record, _| { - let analysis: &AnalysisRecord = &compiled_of(record)?.analysis; - let rows: &[InferredRow] = analysis.inferred.as_deref().unwrap_or_default(); - let mut inferred_columns: Vec<(String, Option)> = - Vec::with_capacity(rows.len()); - for (name, column_type, _) in rows { - let poisoned: bool = self.overlay.poisoned.contains(name); - inferred_columns.push((name.clone(), column_type.clone().filter(|_| !poisoned))); - } - let dynamic_proof: Option = - analysis.proof.as_ref().map(|proof| DynamicProofFact { - output_proven: proof.output_proven, - bare_dynamic_pivot: proof.bare_dynamic_pivot, - families: casefolded_families(&proof.families), - failure_reason: proof.failure_reason.clone(), - named_families: proof.families.clone(), - }); - let fact: ModelAnalysisFact = ModelAnalysisFact { - inferred_columns, - dynamic_proof, - inferred: analysis.inferred.is_some(), - nullable_columns: rows - .iter() - .map(|(_, _, nullability)| nullability == NULLABLE) - .collect(), - fast_lineage_has_star: analysis.has_star, - unchecked_output_columns: self.overlay.poisoned.clone(), - }; - Ok((model_name(record), fact)) - }) - } - - /// Each output column of the kept lineage with its `(resource name, column)` sources. - pub(crate) fn lineage_inputs(&self, py: Python<'_>) -> PyResult>> { - self.compiled(py, |compiled, _| { - Ok(compiled - .analysis - .lineage_rows() - .map(|rows| rows.iter().map(lineage_input).collect())) - }) - } - - /// Whether the model's dynamic pivot proof proved its output columns. - pub(crate) fn output_proven(&self, py: Python<'_>) -> PyResult { - self.compiled(py, |compiled, _| { - Ok(compiled - .analysis - .proof - .as_ref() - .is_some_and(|proof| proof.output_proven)) - }) - } - - /// Output columns whose types semantic recovery forgot. - pub(crate) fn unchecked_columns(&self) -> &[String] { - &self.overlay.poisoned - } - - /// The kept lineage rows with their transform and confidence codes, or `None`. - pub(crate) fn lineage_rows(&self, py: Python<'_>) -> PyResult>> { - self.compiled(py, |compiled, _| { - Ok(compiled.analysis.lineage_rows().map(<[LineageRow]>::to_vec)) - }) - } - - /// The input relations of a dynamic pivot proof that proved its outputs, else `None`. - pub(crate) fn proven_inputs(&self, py: Python<'_>) -> PyResult>> { - self.compiled(py, |compiled, _| { - Ok(compiled - .analysis - .proof - .as_ref() - .filter(|proof| proof.output_proven) - .map(|proof| proof.input_relations.clone())) - }) - } - - /// The attached schema's declared `SchemaColumn`s, or `None` without a schema. - pub(crate) fn declared_columns(&self, py: Python<'_>) -> PyResult>> { - self.read(py, |record, py| { - record - .schema - .as_ref() - .map(|schema| schema.host.columns.project(py).map(Bound::unbind)) - .transpose() - }) - } - - /// The attached schema's entry audits, its column specs and each audited column's index and name. - pub(crate) fn schema_audits_and_columns( - &self, - py: Python<'_>, - ) -> PyResult> { - read_input_with_owner(&self.owner, self.index, py, |owner, record| { - let (Some(schema), Some(fact)) = (&record.schema, owner.input_schema_fact(record)) - else { - return Ok(None); - }; - let audited: Vec<(usize, String)> = fact - .columns - .iter() - .enumerate() - .filter(|(_, column)| !column.audit_definitions.is_empty()) - .map(|(index, column)| (index, column.name.clone())) - .collect(); - Ok(Some(( - schema.host.audits.clone(), - schema.host.columns.clone(), - audited, - ))) - }) - } - - /// What SQL rules select the model by, plus its cache key inputs when `keyed`. - pub(crate) fn sql_rule_inputs(&self, py: Python<'_>, keyed: bool) -> PyResult { - read_input_with_owner(&self.owner, self.index, py, |owner, record| { - let facts: &CompiledModelFacts = - model_facts(owner.facts(), &record.name).ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") - })?; - let excluded: Option = match &facts.config.encoding_error { - Some(_) => None, - None => Some( - facts.config.values.get(SQL_ANALYSIS_KEY) - == Some(&serde_json::Value::Bool(false)), - ), - }; - let key = keyed - .then(|| -> PyResult { - Ok(SqlRuleKey { - authored_sql: facts.authored_sql.to_string(), - query_sql: facts.query_sql.to_string(), - contract: compiled_of(record)? - .analysis - .proof - .as_ref() - .map(contract_proof_json), - }) - }) - .transpose()?; - Ok(SqlRuleInputs { - posix_path: facts.relative_path.clone(), - excluded, - key, - }) - }) - } - - /// The model's name, `str(relative_path)` and authored file text. - pub(crate) fn file_texts(&self, py: Python<'_>) -> PyResult<(String, String, Arc)> { - self.read(py, |record, py| { - let model_file: &NativeModelFile = record.host.model_file.get(); - Ok(( - model_name(record), - model_file.relative_text(py)?, - Arc::clone(model_file.contents_text()), - )) - }) - } - - /// The name, compiled query and authored file of the model, and whether it calls a function. - pub(crate) fn metadata_texts( - &self, - py: Python<'_>, - function_kinds: &[&str], - ) -> PyResult { - self.read(py, |record, _| { - Ok(ModelMetadataTexts { - name: model_name(record), - query_sql: record.query_sql.to_string(), - authored_sql: record.host.model_file.get().contents_text().to_string(), - calls_functions: record - .references - .iter() - .any(|reference| function_kinds.contains(&reference.kind.as_str())), - }) - }) - } - - /// `relative_path.as_posix()` and the compiled query, which compiled artifacts are written from. - pub(crate) fn output_texts(&self, py: Python<'_>) -> PyResult<(String, Arc)> { - self.read(py, |record, py| { - Ok(( - record.host.model_file.get().posix_path(py)?, - Arc::clone(&record.query_sql), - )) - }) - } - - /// The model's authored file text and expanded query, from its recorded facts. - pub(crate) fn sql_texts(&self, py: Python<'_>) -> PyResult<(String, String)> { - let name: String = self.name(py)?; - let owner = self.owner.bind(py).borrow(); - let facts: &CompiledModelFacts = model_facts(owner.facts(), &name).ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") - })?; - Ok((facts.authored_sql.to_string(), facts.query_sql.to_string())) - } - - fn model_file( - &self, - py: Python<'_>, - read: impl FnOnce(&NativeModelFile) -> PyResult, - ) -> PyResult { - self.read(py, |record, _| read(record.host.model_file.get())) - } -} - -/// A lineage row's output column with its `(resource name, column)` sources. -fn lineage_input(row: &LineageRow) -> LineageInput { - let mut sources: Vec<(String, String)> = Vec::with_capacity(row.sources.len()); - for (_, resource, column) in &row.sources { - sources.push((resource.clone(), column.clone())); - } - (row.output_column.clone(), sources) -} - -/// Each family's name casefolded as Python's, with its inferred type. -fn casefolded_families(families: &[(String, Option)]) -> Vec<(String, Option)> { - families - .iter() - .map(|(name, kind)| (str::to_lowercase(name), kind.clone())) - .collect() -} - -/// The compiled record of an input, which assembly records before any view is handed out. -fn compiled_of(record: &ModelInputRecord) -> PyResult<&CompiledRecord> { - record - .compiled - .as_ref() - .ok_or_else(|| pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled")) -} - -#[pymethods] -impl NativeCompiledModel { - #[getter] - pub(crate) fn key(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| { - let compiled: &CompiledRecord = compiled_of(record)?; - compiled - .host - .key - .get_or_try_init(py, || { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - object_key(&classes.compiled, MODEL_RESOURCE, &model_name(record)) - }) - .map(|key| key.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn deps(&self, py: Python<'_>) -> PyResult> { - self.compiled(py, |compiled, py| { - compiled - .host - .deps - .get_or_try_init(py, || { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - let mut keys: Vec> = Vec::with_capacity(compiled.deps.len()); - for (kind, name) in &compiled.deps { - keys.push(object_key(&classes.compiled, kind, name)?); - } - Ok::<_, pyo3::PyErr>(PyTuple::new(py, keys)?.unbind()) - }) - .map(|deps| deps.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn name(&self, py: Python<'_>) -> PyResult { - self.read(py, |record, _| Ok(model_name(record))) - } - - #[getter] - pub(crate) fn relative_path(&self, py: Python<'_>) -> PyResult> { - self.model_file(py, |file| file.relative_path(py)) - } - - #[getter] - pub(crate) fn query_sql<'py>(&self, py: Python<'py>) -> PyResult> { - self.read(py, |record, py| Ok(query_sql_text(py, record))) - } - - #[getter] - pub(crate) fn config(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.host.config.clone_ref(py))) - } - - #[getter] - pub(crate) fn destination(&self, py: Python<'_>) -> PyResult> { - self.read_destination(py, |destination| { - self.projections - .destination - .get_or_try_init(py, || { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - relation_location(&classes.compiled, destination) - }) - .map(|destination| destination.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn references(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.read(py, |record, _| references(&classes, record)) - } - - /// Each reference as `(kind, name, package, call argument count)`. - #[getter] - pub(crate) fn reference_rows(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, _| { - Ok(record - .references - .iter() - .cloned() - .map(reference_row_tuple) - .collect()) - }) - } - - /// The model's declared key facts, for deriving its unique keys. - #[getter] - pub(crate) fn relation_key_row(&self, py: Python<'_>) -> PyResult { - read_input_with_owner(&self.owner, self.index, py, |owner, record| { - model_relation_key_row(owner, record, py) - }) - } - - /// `(resource type, name)` of the model's key. - #[getter] - pub(crate) fn key_row(&self, py: Python<'_>) -> PyResult<(String, String)> { - self.read(py, |record, _| { - Ok((MODEL_RESOURCE.to_owned(), model_name(record))) - }) - } - - /// `(resource type, name)` of each dependency. - #[getter] - pub(crate) fn dep_rows(&self, py: Python<'_>) -> PyResult> { - self.compiled(py, |compiled, _| Ok(compiled.deps.clone())) - } - - /// `(database, schema, name)` of the model's destination. - #[getter] - pub(crate) fn destination_row(&self, py: Python<'_>) -> PyResult { - self.read_destination(py, |destination| { - Ok(( - destination.database.clone(), - destination.schema.clone(), - destination.name.clone(), - )) - }) - } - - /// Each inferred column as `(name, type, nullability)`, or `None` when none were inferred. - #[getter] - pub(crate) fn inferred_rows(&self, py: Python<'_>) -> PyResult>> { - self.compiled(py, |compiled, _| { - let Some(rows) = &compiled.analysis.inferred else { - return Ok(None); - }; - let mut shown: Vec = Vec::with_capacity(rows.len()); - for (name, column_type, nullability) in rows { - let poisoned: bool = self.overlay.poisoned.contains(name); - let column_type: Option = if poisoned { None } else { column_type.clone() }; - shown.push((name.clone(), column_type, nullability.clone())); - } - Ok(Some(shown)) - }) - } - - /// Each declared column of the attached schema as `(name, type)`, or `None` without one. - #[getter] - pub(crate) fn schema_column_rows(&self, py: Python<'_>) -> PyResult>> { - let name: String = self.name(py)?; - let owner = self.owner.bind(py).borrow(); - let facts: &CompiledModelFacts = model_facts(owner.facts(), &name).ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") - })?; - let Some(schema) = &facts.schema else { - return Ok(None); - }; - Ok(Some( - schema - .columns - .iter() - .map(|column| (column.name.clone(), column.data_type.clone())) - .collect(), - )) - } - - /// Whether the attached schema declares dynamic columns. - #[getter(has_dynamic_columns)] - pub(crate) fn dynamic_columns_declared(&self, py: Python<'_>) -> PyResult { - self.read(py, |record, _py| { - Ok(record - .schema - .as_ref() - .is_some_and(|schema| !schema.host.dynamic_columns.specs.is_empty())) - }) - } - - /// Inferred columns, else the attached schema's declared columns, else zero. - #[getter] - pub(crate) fn output_column_count(&self, py: Python<'_>) -> PyResult { - self.read(py, |record, _py| { - if let Some(rows) = &compiled_of(record)?.analysis.inferred { - return Ok(rows.len()); - } - Ok(record - .schema - .as_ref() - .map_or(0, |schema| schema.host.columns.specs.len())) - }) - } - - #[getter] - pub(crate) fn schema_entry(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.read(py, |record, _| schema_entry(&classes, record)) - } - - #[getter] - pub(crate) fn inferred_columns(&self, py: Python<'_>) -> PyResult> { - self.compiled(py, |compiled, py| { - self.projections - .inferred_columns - .get_or_try_init(py, || { - let Some(rows) = &compiled.analysis.inferred else { - return Ok::<_, pyo3::PyErr>(py.None()); - }; - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - let mut columns: Vec> = Vec::with_capacity(rows.len()); - for row in rows { - let poisoned_row: InferredRow; - let row = if self.overlay.poisoned.contains(&row.0) { - poisoned_row = (row.0.clone(), None, row.2.clone()); - &poisoned_row - } else { - row - }; - columns.push(self.owner.bind(py).borrow().inferred_column( - py, - &classes.compiled, - row, - )?); - } - Ok(PyTuple::new(py, columns)?.into_any().unbind()) - }) - .map(|columns| columns.clone_ref(py)) - }) - } - - /// The upstream source count of the model's lineage facts, `None` without them. - pub(crate) fn lineage_edge_count(&self, py: Python<'_>) -> PyResult> { - self.compiled(py, |compiled, _| Ok(compiled.analysis.lineage_edge_count())) - } - - #[getter] - pub(crate) fn fast_lineage_columns(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.compiled(py, |compiled, py| { - compiled.analysis.lineage_object(py, &classes.compiled) - }) - } - - #[getter] - pub(crate) fn fast_lineage_has_star(&self, py: Python<'_>) -> PyResult { - self.compiled(py, |compiled, _| Ok(compiled.analysis.has_star)) - } - - #[getter] - pub(crate) fn fast_lineage_star_resolved(&self, py: Python<'_>) -> PyResult { - self.compiled(py, |compiled, _| Ok(compiled.analysis.star_resolved)) - } - - #[getter] - pub(crate) fn authored_sql<'py>(&self, py: Python<'py>) -> PyResult> { - self.model_file(py, |file| Ok(file.contents(py))) - } - - #[getter] - pub(crate) fn authored_query_sql<'py>( - &self, - py: Python<'py>, - ) -> PyResult> { - self.model_file(py, |file| Ok(file.query_sql(py))) - } - - #[getter] - pub(crate) fn output_column_locations<'py>( - &self, - py: Python<'py>, - ) -> PyResult> { - self.model_file(py, |file| file.output_column_locations(py)) - } - - #[getter] - pub(crate) fn extract_implicit_alias_columns(&self, py: Python<'_>) -> PyResult { - self.model_file(py, |file| Ok(file.extract_implicit_alias_columns())) - } - - #[getter] - pub(crate) fn macro_deps<'py>(&self, py: Python<'py>) -> PyResult> { - self.compiled(py, |compiled, py| PyTuple::new(py, &compiled.macro_deps)) - } - - #[getter] - pub(crate) fn enum_declarations(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| { - Ok(record.host.enum_declarations.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn constant_declarations(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| { - Ok(record.host.constant_declarations.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn enum_columns(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.host.enum_columns.clone_ref(py))) - } - - #[getter] - pub(crate) fn binding_diagnostics(&self, py: Python<'_>) -> PyResult> { - if let Some(diagnostics) = &self.overlay.binding_diagnostics { - return Ok(diagnostics.clone_ref(py)); - } - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.read(py, |record, py| { - let name: String = model_name(record); - let path: Bound<'_, PyAny> = record.host.model_file.get().path_objects(py)?.1; - compiled_of(record)? - .analysis - .binding_tuple(py, &classes.compiled, (&name, &path)) - }) - } - - #[getter] - pub(crate) fn binding_validated(&self, py: Python<'_>) -> PyResult { - self.compiled(py, |compiled, _| Ok(compiled.analysis.binding_validated)) - } - - #[getter] - pub(crate) fn dynamic_column_contract(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.compiled(py, |compiled, py| { - compiled.analysis.proof_object(py, &classes.compiled) - }) - } - - #[getter] - pub(crate) fn unchecked_output_columns<'py>( - &self, - py: Python<'py>, - ) -> PyResult> { - PyFrozenSet::new(py, &self.overlay.poisoned) - } - - #[getter] - pub(crate) fn rejected_sql_analysis_opt_out(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(rejected_opt_out(record, py))) - } - - /// This model with its final binding diagnostics. - fn with_binding_diagnostics(&self, py: Python<'_>, diagnostics: Py) -> Self { - self.changed(py, |overlay| { - overlay.binding_diagnostics = Some(diagnostics) - }) - } - - /// This model after semantic type recovery: its bindings and the outputs left unchecked. - fn with_type_recovery( - &self, - py: Python<'_>, - diagnostics: Py, - poisoned: Vec, - ) -> Self { - self.changed(py, |overlay| { - overlay.binding_diagnostics = Some(diagnostics); - overlay.poisoned = poisoned; - }) - } - - /// This model with the adapter's default schema and database filled into its destination. - pub(crate) fn with_target_defaults( - &self, - py: Python<'_>, - defaults: (Option, Option), - render: Bound<'_, PyAny>, - ) -> PyResult { - let destination: DestinationFact = self.compiled(py, |compiled, _| { - let current: &DestinationFact = self - .overlay - .destination - .as_ref() - .unwrap_or(&compiled.destination); - defaulted_destination(current, defaults, &render) - })?; - Ok(self.changed(py, |overlay| overlay.destination = Some(destination))) - } -} - -/// The model's project graph row: its key, dependencies, tags and folder. -pub(crate) fn model_graph_row( - model: &NativeCompiledModel, - py: Python<'_>, -) -> PyResult { - model.read(py, |record, py| { - let compiled: &CompiledRecord = compiled_of(record)?; - let values: Bound<'_, PyAny> = native_field(record.host.config.bind(py), VALUES_ATTRIBUTE)?; - let mut tags: Vec = Vec::new(); - if let Some(value) = values.downcast::()?.get_item(TAGS_KEY)? - && (value.is_instance_of::() - || value.is_instance_of::()) - { - for tag in value.try_iter()? { - tags.push(tag?.str()?.extract()?); - } - } - let folder: String = record.host.model_file.get().folder(py)?; - Ok(ResourceRow { - key: (MODEL_RESOURCE.to_owned(), model_name(record)), - deps: compiled.deps.clone(), - tags, - folder: Some(folder), - }) - }) -} - -fn model_name(record: &ModelInputRecord) -> String { - record.name.clone() -} - -/// The `CompiledRelationLocation` of a destination. -pub(crate) fn relation_location( - classes: &CompiledViewClasses<'_>, - destination: &DestinationFact, -) -> PyResult> { - let fields: Bound<'_, PyDict> = PyDict::new(classes.relation_location.py()); - fields.set_item(DATABASE_KEY, &destination.database)?; - fields.set_item(SCHEMA_KEY, &destination.schema)?; - fields.set_item("name", &destination.name)?; - fields.set_item("qualified_name", &destination.qualified_name)?; - fields.set_item(LOGICAL_SCHEMA_ATTRIBUTE, &destination.logical_schema)?; - fields.set_item(LOGICAL_DATABASE_ATTRIBUTE, &destination.logical_database)?; - Ok(classes.relation_location.call((), Some(&fields))?.unbind()) -} - -pub(crate) fn object_key( - classes: &CompiledViewClasses<'_>, - kind: &str, - name: &str, -) -> PyResult> { - let fields: Bound<'_, PyDict> = PyDict::new(classes.object_key.py()); - fields.set_item("resource_type", kind)?; - fields.set_item("name", name)?; - Ok(classes.object_key.call((), Some(&fields))?.unbind()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_reads.rs deleted file mode 100644 index a560c7998d..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_reads.rs +++ /dev/null @@ -1,182 +0,0 @@ -//! A compiled function's attributes the planner reads, from its native view. - -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; - -use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; -use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; -use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -/// The native view behind a function object; any other object is a Python-built record, counted. -#[inline] -fn native_function<'a>( - function: &'a Bound<'_, PyAny>, -) -> PyResult> { - if let Ok(view) = function.cast::() { - return Ok(Some(view.get())); - } - count_stage(function.py(), COMPILED_VIEW_FALLBACKS, 1)?; - Ok(None) -} - -/// The compiled function's `arguments`, as its Python attribute reads it. -pub(crate) fn arguments<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .arguments(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "arguments") -} - -/// The compiled function's `body_sql`, as its Python attribute reads it. -pub(crate) fn body_sql<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .body_sql(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "body_sql") -} - -/// The compiled function's `deps`, as its Python attribute reads it. -pub(crate) fn deps<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view.deps(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "deps") -} - -/// The compiled function's `destination`, as its Python attribute reads it. -pub(crate) fn destination<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .destination(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "destination") -} - -/// The compiled function's `entry_point`, as its Python attribute reads it. -pub(crate) fn entry_point<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .entry_point(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "entry_point") -} - -/// The compiled function's `fingerprint_destination`, as its Python attribute reads it. -pub(crate) fn fingerprint_destination<'py>( - function: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .fingerprint_destination(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "fingerprint_destination") -} - -/// The compiled function's `key`, as its Python attribute reads it. -pub(crate) fn key<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "key") -} - -/// The compiled function's `language`, as its Python attribute reads it. -pub(crate) fn language<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .language(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "language") -} - -/// The compiled function's `name`, as its Python attribute reads it. -pub(crate) fn name<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "name") -} - -/// The compiled function's `packages`, as its Python attribute reads it. -pub(crate) fn packages<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .packages(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "packages") -} - -/// The compiled function's `relative_path`, as its Python attribute reads it. -pub(crate) fn relative_path<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .relative_path(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "relative_path") -} - -/// The compiled function's `return_columns`, as its Python attribute reads it. -pub(crate) fn return_columns<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .return_columns(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "return_columns") -} - -/// The compiled function's `returns`, as its Python attribute reads it. -pub(crate) fn returns<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .returns(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "returns") -} - -/// The compiled function's `runtime_version`, as its Python attribute reads it. -pub(crate) fn runtime_version<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .runtime_version(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "runtime_version") -} - -/// The compiled function's `source_file_path`, as its Python attribute reads it. -pub(crate) fn source_file_path<'py>(function: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = function.py(); - if let Some(view) = native_function(function)? { - return view - .source_file_path(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(function, "source_file_path") -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs deleted file mode 100644 index 9cfb2af044..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs +++ /dev/null @@ -1,374 +0,0 @@ -//! A compile's attached inputs, read from the native compiled project that owns its models. - -use pyo3::exceptions::PyTypeError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyclass, pymethods}; - -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; - -const MODEL_INPUTS_FIELD: &str = "model_inputs"; -const RUN_ID_FIELD: &str = "run_id"; -const NO_SQL_VALIDATION_FIELD: &str = "no_sql_validation"; - -/// Compile inputs held as Python objects: user code, discovery or an integration owns them. -struct BoundaryInputs { - /// The discovered project config. - project_config: Py, - /// The discovered local config. - local_config: Py, - /// The discovered project inputs. - discovered_inputs: Py, - /// The adapter's lexical syntax. - sql_lexical_syntax: Py, - /// The selected target config, or `None`. - effective_target: Py, - /// The compile cache directory path. - compile_cache_dir: Py, - /// The resolved connection, adapter integration values. - effective_connection: Py, - /// Project variables, user values. - effective_vars: Py, - /// The macro context user macro bodies run with. - macro_context: Py, - /// Loaded macro modules, user code. - loaded_macros: Py, - /// Public enums, user declaration values. - public_enums: Py, - /// Public constants, user declaration values. - public_constants: Py, - /// The external reference resolver integration, or `None`. - external_sql_reference_resolver: Py, -} - -/// Compiler-derived inputs still held as Python objects until they are owner facts. -struct CompilerHandles { - /// Effective settings. - effective_settings: Py, - /// Attached seed inputs. - seed_inputs: Py, - /// Attached source inputs. - source_inputs: Py, - /// Attached SQL function inputs. - sql_function_inputs: Py, - /// Attached SQL test inputs. - test_inputs: Py, - /// Attached SQL scenario inputs. - scenario_inputs: Py, - /// Attached audit inputs. - audit_inputs: Py, - /// Compiler diagnostics. - diagnostics: Py, - /// The declaration scope index. - scope_index: Py, - /// The declaration scope build. - declaration_scope: Py, -} - -/// The values of one compile inputs view. -struct InputsFields { - run_id: String, - effective_target_name: Option, - no_sql_validation: bool, - model_inputs: Py, - boundary: BoundaryInputs, - compiler: CompilerHandles, -} - -impl InputsFields { - /// The fields named in `fields`, each required. - fn from_fields(fields: &Bound<'_, PyDict>) -> PyResult { - Ok(Self { - run_id: required(fields, RUN_ID_FIELD)?.extract()?, - effective_target_name: required(fields, "effective_target_name")?.extract()?, - no_sql_validation: required(fields, NO_SQL_VALIDATION_FIELD)?.extract()?, - model_inputs: required(fields, MODEL_INPUTS_FIELD)? - .downcast_into::()? - .unbind(), - boundary: BoundaryInputs { - project_config: required(fields, "project_config")?.unbind(), - local_config: required(fields, "local_config")?.unbind(), - discovered_inputs: required(fields, "discovered_inputs")?.unbind(), - sql_lexical_syntax: required(fields, "sql_lexical_syntax")?.unbind(), - effective_target: required(fields, "effective_target")?.unbind(), - compile_cache_dir: required(fields, "compile_cache_dir")?.unbind(), - effective_connection: required(fields, "effective_connection")?.unbind(), - effective_vars: required(fields, "effective_vars")?.unbind(), - macro_context: required(fields, "macro_context")?.unbind(), - loaded_macros: required(fields, "loaded_macros")?.unbind(), - public_enums: required(fields, "public_enums")?.unbind(), - public_constants: required(fields, "public_constants")?.unbind(), - external_sql_reference_resolver: required( - fields, - "external_sql_reference_resolver", - )? - .unbind(), - }, - compiler: CompilerHandles { - effective_settings: required(fields, "effective_settings")?.unbind(), - seed_inputs: required(fields, "seed_inputs")?.unbind(), - source_inputs: required(fields, "source_inputs")?.unbind(), - sql_function_inputs: required(fields, "sql_function_inputs")?.unbind(), - test_inputs: required(fields, "test_inputs")?.unbind(), - scenario_inputs: required(fields, "scenario_inputs")?.unbind(), - audit_inputs: required(fields, "audit_inputs")?.unbind(), - diagnostics: required(fields, "diagnostics")?.unbind(), - scope_index: required(fields, "scope_index")?.unbind(), - declaration_scope: required(fields, "declaration_scope")?.unbind(), - }, - }) - } - - /// Every field by name. - fn as_fields<'py>(&self, py: Python<'py>) -> PyResult> { - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item(RUN_ID_FIELD, &self.run_id)?; - fields.set_item("effective_target_name", &self.effective_target_name)?; - fields.set_item(NO_SQL_VALIDATION_FIELD, self.no_sql_validation)?; - fields.set_item(MODEL_INPUTS_FIELD, self.model_inputs.bind(py))?; - fields.set_item("project_config", self.boundary.project_config.bind(py))?; - fields.set_item("local_config", self.boundary.local_config.bind(py))?; - fields.set_item( - "discovered_inputs", - self.boundary.discovered_inputs.bind(py), - )?; - fields.set_item( - "sql_lexical_syntax", - self.boundary.sql_lexical_syntax.bind(py), - )?; - fields.set_item("effective_target", self.boundary.effective_target.bind(py))?; - fields.set_item( - "compile_cache_dir", - self.boundary.compile_cache_dir.bind(py), - )?; - fields.set_item( - "effective_connection", - self.boundary.effective_connection.bind(py), - )?; - fields.set_item("effective_vars", self.boundary.effective_vars.bind(py))?; - fields.set_item("macro_context", self.boundary.macro_context.bind(py))?; - fields.set_item("loaded_macros", self.boundary.loaded_macros.bind(py))?; - fields.set_item("public_enums", self.boundary.public_enums.bind(py))?; - fields.set_item("public_constants", self.boundary.public_constants.bind(py))?; - fields.set_item( - "external_sql_reference_resolver", - self.boundary.external_sql_reference_resolver.bind(py), - )?; - fields.set_item( - "effective_settings", - self.compiler.effective_settings.bind(py), - )?; - fields.set_item("seed_inputs", self.compiler.seed_inputs.bind(py))?; - fields.set_item("source_inputs", self.compiler.source_inputs.bind(py))?; - fields.set_item( - "sql_function_inputs", - self.compiler.sql_function_inputs.bind(py), - )?; - fields.set_item("test_inputs", self.compiler.test_inputs.bind(py))?; - fields.set_item("scenario_inputs", self.compiler.scenario_inputs.bind(py))?; - fields.set_item("audit_inputs", self.compiler.audit_inputs.bind(py))?; - fields.set_item("diagnostics", self.compiler.diagnostics.bind(py))?; - fields.set_item("scope_index", self.compiler.scope_index.bind(py))?; - fields.set_item( - "declaration_scope", - self.compiler.declaration_scope.bind(py), - )?; - Ok(fields) - } -} - -fn required<'py>(fields: &Bound<'py, PyDict>, name: &str) -> PyResult> { - fields - .get_item(name)? - .ok_or_else(|| PyTypeError::new_err(format!("missing compile input field {name}"))) -} - -/// One compile's attached inputs, its model inputs projected from the owner. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeInputsView { - owner: Py, - fields: InputsFields, -} - -impl NativeInputsView { - /// The view over `owner` with these input fields. - pub(crate) fn new( - owner: Py, - fields: &Bound<'_, PyDict>, - ) -> PyResult { - Ok(Self { - owner, - fields: InputsFields::from_fields(fields)?, - }) - } -} - -#[pymethods] -impl NativeInputsView { - /// These inputs with `changes` applied; the owner's model inputs are shared. - #[pyo3(signature = (**changes))] - fn with_changes(&self, py: Python<'_>, changes: Option>) -> PyResult { - let current: Bound<'_, PyDict> = self.fields.as_fields(py)?; - if let Some(changes) = changes { - for (name, value) in changes.iter() { - if !current.contains(&name)? { - return Err(PyTypeError::new_err(format!( - "unknown compile input field {name}" - ))); - } - current.set_item(name, value)?; - } - } - Self::new(self.owner.clone_ref(py), ¤t) - } - - /// Every field by name, for an explicit `CompileProjectInputs` projection. - fn fields<'py>(&self, py: Python<'py>) -> PyResult> { - let fields: Bound<'py, PyDict> = self.fields.as_fields(py)?; - fields.set_item("native_project", self.owner.bind(py))?; - Ok(fields) - } - - #[getter] - pub(crate) fn run_id(&self) -> String { - self.fields.run_id.clone() - } - - #[getter] - pub(crate) fn effective_target_name(&self) -> Option { - self.fields.effective_target_name.clone() - } - - #[getter(no_sql_validation)] - pub(crate) fn is_sql_validation_skipped(&self) -> bool { - self.fields.no_sql_validation - } - - #[getter] - pub(crate) fn model_inputs(&self, py: Python<'_>) -> Py { - self.fields.model_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn native_project(&self, py: Python<'_>) -> Py { - self.owner.clone_ref(py) - } - - #[getter] - pub(crate) fn project_config(&self, py: Python<'_>) -> Py { - self.fields.boundary.project_config.clone_ref(py) - } - - #[getter] - pub(crate) fn local_config(&self, py: Python<'_>) -> Py { - self.fields.boundary.local_config.clone_ref(py) - } - - #[getter] - pub(crate) fn discovered_inputs(&self, py: Python<'_>) -> Py { - self.fields.boundary.discovered_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn sql_lexical_syntax(&self, py: Python<'_>) -> Py { - self.fields.boundary.sql_lexical_syntax.clone_ref(py) - } - - #[getter] - pub(crate) fn effective_target(&self, py: Python<'_>) -> Py { - self.fields.boundary.effective_target.clone_ref(py) - } - - #[getter] - pub(crate) fn compile_cache_dir(&self, py: Python<'_>) -> Py { - self.fields.boundary.compile_cache_dir.clone_ref(py) - } - - #[getter] - pub(crate) fn effective_connection(&self, py: Python<'_>) -> Py { - self.fields.boundary.effective_connection.clone_ref(py) - } - - #[getter] - pub(crate) fn effective_vars(&self, py: Python<'_>) -> Py { - self.fields.boundary.effective_vars.clone_ref(py) - } - - #[getter] - pub(crate) fn macro_context(&self, py: Python<'_>) -> Py { - self.fields.boundary.macro_context.clone_ref(py) - } - - #[getter] - pub(crate) fn loaded_macros(&self, py: Python<'_>) -> Py { - self.fields.boundary.loaded_macros.clone_ref(py) - } - - #[getter] - pub(crate) fn public_enums(&self, py: Python<'_>) -> Py { - self.fields.boundary.public_enums.clone_ref(py) - } - - #[getter] - pub(crate) fn public_constants(&self, py: Python<'_>) -> Py { - self.fields.boundary.public_constants.clone_ref(py) - } - - #[getter] - pub(crate) fn external_sql_reference_resolver(&self, py: Python<'_>) -> Py { - self.fields - .boundary - .external_sql_reference_resolver - .clone_ref(py) - } - - #[getter] - pub(crate) fn effective_settings(&self, py: Python<'_>) -> Py { - self.fields.compiler.effective_settings.clone_ref(py) - } - - #[getter] - pub(crate) fn seed_inputs(&self, py: Python<'_>) -> Py { - self.fields.compiler.seed_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn source_inputs(&self, py: Python<'_>) -> Py { - self.fields.compiler.source_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn sql_function_inputs(&self, py: Python<'_>) -> Py { - self.fields.compiler.sql_function_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn test_inputs(&self, py: Python<'_>) -> Py { - self.fields.compiler.test_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn scenario_inputs(&self, py: Python<'_>) -> Py { - self.fields.compiler.scenario_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn audit_inputs(&self, py: Python<'_>) -> Py { - self.fields.compiler.audit_inputs.clone_ref(py) - } - - #[getter] - pub(crate) fn diagnostics(&self, py: Python<'_>) -> Py { - self.fields.compiler.diagnostics.clone_ref(py) - } - - #[getter] - pub(crate) fn scope_index(&self, py: Python<'_>) -> Py { - self.fields.compiler.scope_index.clone_ref(py) - } - - #[getter] - pub(crate) fn declaration_scope(&self, py: Python<'_>) -> Py { - self.fields.compiler.declaration_scope.clone_ref(py) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs index 8bff8fd13e..10fcd4e9e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs @@ -1,22 +1,2 @@ //! Python views of the compiled project: model, input, resource and project views and their records. -pub(crate) mod audit_reads; -pub(crate) mod audit_views; -pub(crate) mod compiled_model_analysis; -pub(crate) mod compiled_model_view; -pub(crate) mod function_reads; -pub(crate) mod function_views; -pub(crate) mod getters; -pub(crate) mod inputs_view; -pub(crate) mod model_input_view; -pub(crate) mod model_reads; -pub(crate) mod project_host; -pub(crate) mod project_reads; -pub(crate) mod project_view; -pub(crate) mod records; pub(crate) mod registration; -pub(crate) mod relation_keys; -pub(crate) mod resource_reads; -pub(crate) mod seed_reads; -pub(crate) mod seed_views; -pub(crate) mod source_reads; -pub(crate) mod source_views; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs deleted file mode 100644 index df851fe4ee..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs +++ /dev/null @@ -1,604 +0,0 @@ -//! A model's compile input, read from the native compiled project that owns it. - -use sqlbuild_project::models as project_models; -use sqlbuild_project::models::ExpansionRecord; -use std::sync::Arc; - -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{FromPyObject, IntoPyObject, pyclass, pymethods}; -use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; -use sqlbuild_analysis::sql_macros::main::typed_analysis_sql::typed_analysis_sql; -use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; - -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::rows::{ReferenceTuple, reference_row_tuple}; -use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - CompiledArguments, CompiledViewClasses, NativeCompiledModel, compiled_record, -}; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::compiled_views::relation_keys::{RelationKeyRow, relation_key_row}; -use sqlbuild_model_config::header_metadata::models::DynamicFamily; -use sqlbuild_project::models::ReferenceRow; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::discovery::main::records::projected_text::projected_text; -use sqlbuild_python_front::discovery::models::NativeModelFile; -use sqlbuild_python_front::model_config::main::headers::family_specs::family_specs; -use sqlbuild_python_front::model_config::models::HeaderAudits; -use sqlbuild_python_front::model_config::models::HeaderColumns; -use sqlbuild_python_front::model_config::models::HeaderFamilies; -use sqlbuild_python_front::render::main::calls::span_tuple::span_tuple; -use sqlbuild_python_front::render::types::SpanRow; - -const MODEL_RELATION_KIND: &str = "ref"; - -/// The Python classes a model input's projected attributes are built with. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -pub(crate) struct InputViewClasses<'py> { - pub(crate) compiled: CompiledViewClasses<'py>, - sql_reference: Bound<'py, PyAny>, - reference_kind: Bound<'py, PyAny>, - schema_model_entry: Bound<'py, PyAny>, - sql_expansion: Bound<'py, PyAny>, - expansion_span: Bound<'py, PyAny>, -} - -/// A model's attached header schema; the host keeps its columns, audits and dynamic columns. -pub(crate) type SchemaRecord = project_models::SchemaRecord; - -/// What the model loop finished for one model beyond its facts. -pub(crate) type ModelInputRecord = project_models::ModelInputRecord; - -/// A model schema's header columns and specs, which carry user-authored audit arguments. -pub(crate) struct SchemaHandles { - pub(crate) columns: Arc, - pub(crate) audits: Arc, - pub(crate) dynamic_columns: Arc, -} - -/// A model input's user values and the attribute projections its views build. -pub(crate) struct ModelInputHandles { - /// The discovered model file, owned by the discovery inputs. - pub(crate) model_file: Py, - /// The resolved config, whose values hold user values. - pub(crate) config: Py, - pub(crate) rejected_opt_out: Option>, - /// The model's own enums and constants, whose values are user values. - pub(crate) enum_declarations: Py, - pub(crate) constant_declarations: Py, - pub(crate) enum_columns: Py, - /// Usage records, the identities the scope index is built from. - pub(crate) macro_usages: Py, - pub(crate) declaration_usages: Py, - /// Attribute projections built on first access, shared by every view of the model. - pub(crate) projections: InputProjections, -} - -/// A model input's Python attribute projections, each built once on first access. -pub(crate) struct InputProjections { - query_sql: PyOnceLock>, - references: PyOnceLock>, - schema_entry: PyOnceLock>, - sql_expansion: PyOnceLock>, - lint_expansion: PyOnceLock>, -} - -impl Default for InputProjections { - fn default() -> Self { - Self { - query_sql: PyOnceLock::new(), - references: PyOnceLock::new(), - schema_entry: PyOnceLock::new(), - sql_expansion: PyOnceLock::new(), - lint_expansion: PyOnceLock::new(), - } - } -} - -/// One model's compile input, projected from the owner on demand. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeModelInput { - owner: Py, - index: usize, -} - -impl NativeModelInput { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } - - pub(crate) fn read<'py, T>( - &self, - py: Python<'py>, - read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, - ) -> PyResult { - read_input(&self.owner, self.index, py, read) - } - - /// Read the input record together with the project that owns it. - pub(crate) fn read_with_owner<'py, T>( - &self, - py: Python<'py>, - read: impl FnOnce(&NativeCompiledProject, &ModelInputRecord) -> PyResult, - ) -> PyResult { - read_input_with_owner(&self.owner, self.index, py, read) - } -} - -/// Read the input record at `index` of `owner` together with `owner`. -pub(crate) fn read_input_with_owner( - owner: &Py, - index: usize, - py: Python<'_>, - read: impl FnOnce(&NativeCompiledProject, &ModelInputRecord) -> PyResult, -) -> PyResult { - let owner = owner.bind(py).borrow(); - read(&owner, owner.input(index)?) -} - -/// Read the input record at `index` of `owner`. -pub(crate) fn read_input<'py, T>( - owner: &Py, - index: usize, - py: Python<'py>, - read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, -) -> PyResult { - let owner = owner.bind(py).borrow(); - read(owner.input(index)?, py) -} - -/// The classes the owner's views project attributes with. -pub(crate) fn view_classes<'py>( - owner: &Py, - py: Python<'py>, -) -> PyResult> { - owner.bind(py).borrow().input_classes(py)?.extract() -} - -#[pymethods] -impl NativeModelInput { - #[getter] - pub(crate) fn model_file(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| { - Ok(record.host.model_file.clone_ref(py).into_any()) - }) - } - - #[getter] - pub(crate) fn config(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.host.config.clone_ref(py))) - } - - #[getter] - pub(crate) fn query_sql<'py>(&self, py: Python<'py>) -> PyResult> { - self.read(py, |record, py| Ok(query_sql_text(py, record))) - } - - #[getter] - pub(crate) fn macro_source_sql<'py>(&self, py: Python<'py>) -> PyResult> { - self.read(py, |record, py| { - Ok(PyString::new(py, &record.macro_source_sql)) - }) - } - - #[getter] - pub(crate) fn references(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.read(py, |record, _| references(&classes, record)) - } - - #[getter] - pub(crate) fn schema_entry(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.read(py, |record, _| schema_entry(&classes, record)) - } - - /// Each reference as `(kind, name, package, call argument count)`. - #[getter] - pub(crate) fn reference_rows(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, _| { - Ok(record - .references - .iter() - .cloned() - .map(reference_row_tuple) - .collect()) - }) - } - - /// The model's declared key facts, for deriving its unique keys. - #[getter] - pub(crate) fn relation_key_row(&self, py: Python<'_>) -> PyResult { - self.read_with_owner(py, |owner, record| { - model_relation_key_row(owner, record, py) - }) - } - - /// The name of the schema entry attached to the model, or `None`. - #[getter] - pub(crate) fn attached_schema_name(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, _| Ok(attached_schema_name(record))) - } - - #[getter] - pub(crate) fn schema_file(&self, py: Python<'_>) -> Py { - py.None() - } - - #[getter] - pub(crate) fn sql_validation_enabled(&self, py: Python<'_>) -> PyResult { - self.read(py, |record, _| Ok(record.sql_validation_enabled)) - } - - #[getter] - pub(crate) fn rejected_sql_analysis_opt_out(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(rejected_opt_out(record, py))) - } - - #[getter] - pub(crate) fn enum_declarations(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| { - Ok(record.host.enum_declarations.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn constant_declarations(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| { - Ok(record.host.constant_declarations.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn enum_columns(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.host.enum_columns.clone_ref(py))) - } - - #[getter] - pub(crate) fn macro_deps<'py>(&self, py: Python<'py>) -> PyResult> { - self.read(py, |record, py| PyTuple::new(py, &record.macro_deps)) - } - - #[getter] - pub(crate) fn macro_usages(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.host.macro_usages.clone_ref(py))) - } - - #[getter] - pub(crate) fn declaration_usages(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| { - Ok(record.host.declaration_usages.clone_ref(py)) - }) - } - - /// Record what assembly finished for the model; return its compiled model view. - fn compiled_model( - &self, - py: Python<'_>, - arguments: CompiledArguments, - ) -> PyResult { - let mut owner = self.owner.bind(py).borrow_mut(); - let record: &mut ModelInputRecord = owner.input_mut(self.index)?; - let name: String = record.name.clone(); - record.compiled = Some(compiled_record(py, record, &name, arguments)?); - Ok(NativeCompiledModel::new( - self.owner.clone_ref(py), - self.index, - )) - } - - /// The authored-to-expanded passes, when interpolation left the authored query unchanged. - #[getter] - pub(crate) fn sql_expansion(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.read(py, |record, _| authored_expansion(&classes, record)) - } - - /// The whole expansion, variables included, which early SQL lint reuses. - #[getter] - pub(crate) fn lint_expansion(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.read(py, |record, py| { - record - .host - .projections - .lint_expansion - .get_or_try_init(py, || { - let expansion: &ExpansionRecord = &record.expansion; - expansion_object( - &classes, - expansion, - &[ - &expansion.interpolation_spans, - &expansion.declaration_spans, - &expansion.macro_spans, - ], - ) - }) - .map(|expansion| expansion.clone_ref(py)) - }) - } -} - -/// The compiled query as the Python string every reader shares; the authored query's when equal. -pub(crate) fn query_sql_text<'py>( - py: Python<'py>, - record: &ModelInputRecord, -) -> Bound<'py, PyString> { - let model_file: &NativeModelFile = record.host.model_file.get(); - if Arc::ptr_eq(&record.query_sql, model_file.query_text()) { - return model_file.query_sql(py); - } - projected_text(py, &record.host.projections.query_sql, &record.query_sql) -} - -/// The authored-to-expanded passes, when interpolation left the authored query unchanged. -pub(crate) fn authored_expansion( - classes: &InputViewClasses<'_>, - record: &ModelInputRecord, -) -> PyResult> { - let py: Python<'_> = classes.sql_expansion.py(); - record - .host - .projections - .sql_expansion - .get_or_try_init(py, || { - let expansion: &ExpansionRecord = &record.expansion; - if !expansion.authored { - return Ok(py.None()); - } - expansion_object( - classes, - expansion, - &[&expansion.declaration_spans, &expansion.macro_spans], - ) - }) - .map(|expansion| expansion.clone_ref(py)) -} - -/// Where a rejected `sql_analysis false` was authored, or `None`. -pub(crate) fn rejected_opt_out(record: &ModelInputRecord, py: Python<'_>) -> Py { - record - .host - .rejected_opt_out - .as_ref() - .map_or_else(|| py.None(), |location| location.clone_ref(py)) -} - -/// The model's `CompileSqlReference` tuple, built once. -pub(crate) fn references( - classes: &InputViewClasses<'_>, - record: &ModelInputRecord, -) -> PyResult> { - reference_tuple( - classes, - &record.references, - &record.host.projections.references, - ) -} - -/// `CompileSqlReference` records of `rows`, built once into `cache`. -pub(crate) fn reference_tuple( - classes: &InputViewClasses<'_>, - rows: &[ReferenceRow], - cache: &PyOnceLock>, -) -> PyResult> { - let py: Python<'_> = classes.sql_reference.py(); - cache - .get_or_try_init(py, || { - let mut references: Vec> = Vec::with_capacity(rows.len()); - for ReferenceRow { - kind, - name, - package, - call_argument_count: count, - } in rows - { - let fields: Bound<'_, PyDict> = PyDict::new(py); - fields.set_item("ref_kind", classes.reference_kind.call1((kind,))?)?; - fields.set_item("ref_name", name)?; - fields.set_item("ref_package", package)?; - fields.set_item("call_argument_count", count)?; - references.push(classes.sql_reference.call((), Some(&fields))?); - } - Ok::<_, pyo3::PyErr>(PyTuple::new(py, references)?.unbind()) - }) - .map(|references| references.clone_ref(py)) -} - -/// The model's declared key facts: its config and attached schema columns and audits. -pub(crate) fn model_relation_key_row( - owner: &NativeCompiledProject, - record: &ModelInputRecord, - py: Python<'_>, -) -> PyResult { - let values: Bound<'_, PyAny> = native_field(record.host.config.bind(py), "values")?; - let declared = match (&record.schema, owner.input_schema_fact(record)) { - (Some(schema), Some(fact)) => { - Some((fact.columns.as_slice(), schema.host.audits.key_rows(py)?)) - } - _ => None, - }; - relation_key_row( - MODEL_RELATION_KIND, - record.name.clone(), - values.downcast::()?, - declared, - ) -} - -/// The name of the model's attached schema entry, its file stem, or `None`. -pub(crate) fn attached_schema_name(record: &ModelInputRecord) -> Option { - record.schema.as_ref().map(|_| record.name.clone()) -} - -/// The model's attached `SchemaModelEntry`, or `None`, built once. -pub(crate) fn schema_entry( - classes: &InputViewClasses<'_>, - record: &ModelInputRecord, -) -> PyResult> { - let py: Python<'_> = classes.schema_model_entry.py(); - record - .host - .projections - .schema_entry - .get_or_try_init(py, || { - let Some(schema) = &record.schema else { - return Ok::<_, pyo3::PyErr>(py.None()); - }; - let fields: Bound<'_, PyDict> = PyDict::new(py); - fields.set_item("name", &record.name)?; - fields.set_item("model_schema", &schema.model_schema)?; - fields.set_item("description", &schema.description)?; - fields.set_item("type_enforcement", schema.type_enforcement.then_some(true))?; - fields.set_item("columns", schema.host.columns.project(py)?)?; - fields.set_item("dynamic_columns", schema.host.dynamic_columns.project(py)?)?; - fields.set_item("audits", schema.host.audits.project(py)?)?; - Ok(classes.schema_model_entry.call((), Some(&fields))?.unbind()) - }) - .map(|entry| entry.clone_ref(py)) -} - -/// `CompiledSqlExpansion(authored_sql, expanded_sql, passes)` for these passes. -fn expansion_object( - classes: &InputViewClasses<'_>, - expansion: &ExpansionRecord, - passes: &[&Vec], -) -> PyResult> { - let py: Python<'_> = classes.sql_expansion.py(); - let mut pass_objects: Vec> = Vec::with_capacity(passes.len()); - for spans in passes { - let mut span_objects: Vec> = Vec::with_capacity(spans.len()); - for span in spans.iter() { - span_objects.push( - classes - .expansion_span - .call1(span_tuple(span).into_pyobject(py)?)?, - ); - } - pass_objects.push(PyTuple::new(py, span_objects)?); - } - let fields: Bound<'_, PyDict> = PyDict::new(py); - fields.set_item("authored_sql", &*expansion.authored_sql)?; - fields.set_item("expanded_sql", &*expansion.expanded_sql)?; - fields.set_item("passes", PyTuple::new(py, pass_objects)?)?; - Ok(classes.sql_expansion.call((), Some(&fields))?.unbind()) -} - -/// A model input's attached schema: its name and dynamic column families. -pub(crate) struct AttachedSchema { - pub(crate) name: String, - pub(crate) dynamic_columns: Vec, - pub(crate) type_enforcement: bool, -} - -/// The attached schema of `model_input`; a native input answers without building its entry. -pub(crate) fn attached_schema<'py>( - model_input: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = model_input.py(); - if let Ok(native) = model_input.cast::() { - return native.get().read(py, |record, py| { - let Some(schema) = &record.schema else { - return Ok(None); - }; - Ok(Some(AttachedSchema { - name: record.name.clone(), - dynamic_columns: schema.host.dynamic_columns.read(py)?, - type_enforcement: schema.type_enforcement, - })) - }); - } - let entry: Bound<'py, PyAny> = native_field(model_input, "schema_entry")?; - if entry.is_none() { - return Ok(None); - } - Ok(Some(AttachedSchema { - name: native_field(&entry, "name")?.extract()?, - dynamic_columns: family_specs(&native_field(&entry, "dynamic_columns")?)?, - type_enforcement: native_field(&entry, "type_enforcement")?.is_truthy()?, - })) -} - -/// The declared column facts of `model_input`'s attached schema, empty without one. -pub(crate) fn attached_columns(model_input: &Bound<'_, PyAny>) -> PyResult> { - let py: Python<'_> = model_input.py(); - if let Ok(native) = model_input.cast::() { - return native.get().read_with_owner(py, |owner, record| { - Ok(owner - .input_schema_fact(record) - .map(|fact| fact.columns.clone()) - .unwrap_or_default()) - }); - } - let entry: Bound<'_, PyAny> = native_field(model_input, "schema_entry")?; - if entry.is_none() { - return Ok(Vec::new()); - } - python_column_facts(&native_field(&entry, "columns")?, None) -} - -/// The name of `model_input`, its file stem; a native input answers from its record. -pub(crate) fn input_model_name(model_input: &Bound<'_, PyAny>) -> PyResult { - if let Ok(native) = model_input.cast::() { - return native - .get() - .read(model_input.py(), |record, _| Ok(record.name.clone())); - } - native_field(model_input, "model_file")? - .getattr("file_path")? - .getattr("stem")? - .extract() -} - -/// What the analysed SQL of a native model input must show for its SQL macro calls to fit. -pub(crate) fn input_sql_macro_checks( - model_input: &Bound<'_, PyAny>, -) -> PyResult> { - if let Ok(native) = model_input.cast::() { - return native.get().read(model_input.py(), |record, _| { - Ok(record.sql_macro_checks.clone()) - }); - } - Ok(Vec::new()) -} - -/// The analysis text of `model_input`: its compiled query with each typed SQL macro call marked. -pub(crate) fn input_analysis_sql( - model_input: &Bound<'_, PyAny>, - dialect: &str, -) -> PyResult { - if let Ok(native) = model_input.cast::() { - return native.get().read(model_input.py(), |record, _| { - Ok(typed_analysis_sql(&record.query_sql, &record.typed_calls, dialect).0) - }); - } - native_field(model_input, "query_sql")?.extract() -} - -/// Each reference of `model_input` as `(kind, name, package, call argument count)`. -pub(crate) fn input_reference_rows(model_input: &Bound<'_, PyAny>) -> PyResult> { - let py: Python<'_> = model_input.py(); - if let Ok(native) = model_input.cast::() { - return native - .get() - .read(py, |record, _| Ok(record.references.clone())); - } - native_field(model_input, "references")? - .try_iter()? - .map(|reference| reference_row_of(&reference?)) - .collect() -} - -/// One `CompileSqlReference` record as its row. -fn reference_row_of(reference: &Bound<'_, PyAny>) -> PyResult { - Ok(ReferenceRow { - kind: native_field(reference, "ref_kind")?.str()?.extract()?, - name: native_field(reference, "ref_name")?.extract()?, - package: native_field(reference, "ref_package")?.extract()?, - call_argument_count: native_field(reference, "call_argument_count")?.extract()?, - }) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs deleted file mode 100644 index 93e1daa526..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs +++ /dev/null @@ -1,326 +0,0 @@ -//! A compiled model's analysis outcomes as Rust data: views read their owner, records their fields. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; - -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - DeclaredRow, LineageInput, NativeCompiledModel, -}; -use sqlbuild_project::types::InferredRow; -use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; -use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -/// One lineage output with its transform kind and confidence values, and its sources. -pub(crate) struct LineageFactRow { - pub(crate) output_column: String, - pub(crate) transform: String, - pub(crate) confidence: String, - /// `(resource type, resource, column)` of each source. - pub(crate) sources: Vec<(String, String, String)>, -} - -const TRANSFORM_VALUES: [&str; 6] = [ - "direct", - "cast", - "expression", - "aggregation", - "star", - "constant", -]; -const CONFIDENCE_VALUES: [&str; 3] = ["unknown", "high", "medium"]; - -/// The native view behind a model object; any other object is a Python-built record, counted. -#[inline] -pub(crate) fn native_model<'a>( - model: &'a Bound<'_, PyAny>, -) -> PyResult> { - if let Ok(view) = model.cast::() { - return Ok(Some(view.get())); - } - count_stage(model.py(), COMPILED_VIEW_FALLBACKS, 1)?; - Ok(None) -} - -/// Each inferred output column as `(name, type, nullability)`, `None` when none were inferred. -pub(crate) fn inferred_rows(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Some(view) = native_model(model)? { - return view.inferred_rows(model.py()); - } - let rows: Bound<'_, PyAny> = native_field(model, "inferred_rows")?; - if rows.is_none() { - return Ok(None); - } - let mut inferred: Vec = Vec::new(); - for row in rows.try_iter()? { - let (name, column_type, nullability): (String, Option, Bound<'_, PyAny>) = - row?.extract()?; - inferred.push((name, column_type, nullability.str()?.extract()?)); - } - Ok(Some(inferred)) -} - -/// The names of the inferred output columns, `None` when the shape is unknown. -pub(crate) fn inferred_names(model: &Bound<'_, PyAny>) -> PyResult>> { - Ok(inferred_rows(model)?.map(column_names)) -} - -fn column_names(rows: Vec) -> Vec { - rows.into_iter().map(|(name, _, _)| name).collect() -} - -/// Each declared column of the attached schema as `(name, type)`, `None` without a schema. -pub(crate) fn declared_rows(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Some(view) = native_model(model)? { - return view.schema_column_rows(model.py()); - } - native_field(model, "schema_column_rows")?.extract() -} - -/// Whether the attached schema declares dynamic column families. -pub(crate) fn dynamic_columns_declared(model: &Bound<'_, PyAny>) -> PyResult { - if let Some(view) = native_model(model)? { - return view.dynamic_columns_declared(model.py()); - } - native_field(model, "has_dynamic_columns")?.is_truthy() -} - -/// `(has star, star resolved)` of the model's analysed query. -pub(crate) fn star_flags(model: &Bound<'_, PyAny>) -> PyResult<(bool, bool)> { - if let Some(view) = native_model(model)? { - return Ok(( - view.fast_lineage_has_star(model.py())?, - view.fast_lineage_star_resolved(model.py())?, - )); - } - Ok(( - native_field(model, "fast_lineage_has_star")?.is_truthy()?, - native_field(model, "fast_lineage_star_resolved")?.is_truthy()?, - )) -} - -/// Output columns whose types semantic recovery forgot, in any order. -pub(crate) fn unchecked_columns(model: &Bound<'_, PyAny>) -> PyResult> { - if let Some(view) = native_model(model)? { - return Ok(view.unchecked_columns().to_vec()); - } - native_field(model, "unchecked_output_columns")? - .try_iter()? - .map(|column| column?.extract()) - .collect() -} - -/// Each lineage output with the `(resource, column)` pairs it reads; empty without lineage. -pub(crate) fn lineage_inputs(model: &Bound<'_, PyAny>) -> PyResult> { - if let Some(view) = native_model(model)? { - return Ok(view.lineage_inputs(model.py())?.unwrap_or_default()); - } - let columns: Bound<'_, PyAny> = native_field(model, "fast_lineage_columns")?; - if !columns.is_truthy()? { - return Ok(Vec::new()); - } - let mut lineage: Vec = Vec::new(); - for output in columns.try_iter()? { - let output: Bound<'_, PyAny> = output?; - let mut upstream: Vec<(String, String)> = Vec::new(); - for source in native_field(&output, "upstream_columns")?.try_iter()? { - let source: Bound<'_, PyAny> = source?; - upstream.push(( - native_field(&source, "resource_name")?.extract()?, - native_field(&source, "column_name")?.extract()?, - )); - } - lineage.push((native_field(&output, "output_column")?.extract()?, upstream)); - } - Ok(lineage) -} - -/// Whether the native session validated the model's bindings. -pub(crate) fn binding_validated(model: &Bound<'_, PyAny>) -> PyResult { - if let Some(view) = native_model(model)? { - return view.binding_validated(model.py()); - } - native_field(model, "binding_validated")?.is_truthy() -} - -/// Whether the model's dynamic pivot proof proved its output columns. -pub(crate) fn output_proven(model: &Bound<'_, PyAny>) -> PyResult { - if let Some(view) = native_model(model)? { - return view.output_proven(model.py()); - } - let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?; - Ok(!proof.is_none() && native_field(&proof, "output_proven")?.is_truthy()?) -} - -/// Each lineage output with its transform kind and confidence values, `None` without lineage. -pub(crate) fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Some(view) = native_model(model)? { - let Some(rows) = view.lineage_rows(model.py())? else { - return Ok(None); - }; - let mut facts: Vec = Vec::with_capacity(rows.len()); - for row in rows { - facts.push(LineageFactRow { - output_column: row.output_column, - transform: coded(&TRANSFORM_VALUES, row.transform_code)?, - confidence: coded(&CONFIDENCE_VALUES, row.confidence_code)?, - sources: row.sources, - }); - } - return Ok(Some(facts)); - } - let columns: Bound<'_, PyAny> = native_field(model, "fast_lineage_columns")?; - if columns.is_none() { - return Ok(None); - } - let mut facts: Vec = Vec::new(); - for output in columns.try_iter()? { - let output: Bound<'_, PyAny> = output?; - let mut sources: Vec<(String, String, String)> = Vec::new(); - for source in native_field(&output, "upstream_columns")?.try_iter()? { - let source: Bound<'_, PyAny> = source?; - sources.push(( - native_field(&source, "resource_type")?.extract()?, - native_field(&source, "resource_name")?.extract()?, - native_field(&source, "column_name")?.extract()?, - )); - } - facts.push(LineageFactRow { - output_column: native_field(&output, "output_column")?.extract()?, - transform: native_field(&output, "transform_kind")?.str()?.extract()?, - confidence: native_field(&output, "confidence")?.str()?.extract()?, - sources, - }); - } - Ok(Some(facts)) -} - -fn coded(values: &[&str], code: u8) -> PyResult { - values - .get(usize::from(code)) - .map(|value| (*value).to_owned()) - .ok_or_else(|| pyo3::exceptions::PyIndexError::new_err("tuple index out of range")) -} - -/// The input relations of a dynamic pivot proof that proved the model's outputs, else `None`. -pub(crate) fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Some(view) = native_model(model)? { - return view.proven_inputs(model.py()); - } - let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?; - if proof.is_none() || !native_field(&proof, "output_proven")?.is_truthy()? { - return Ok(None); - } - native_field(&proof, "input_relations")?.extract().map(Some) -} - -/// The model's name. -pub(crate) fn model_name(model: &Bound<'_, PyAny>) -> PyResult { - if let Some(view) = native_model(model)? { - return view.name(model.py()); - } - native_field(model, "name")?.extract() -} - -/// The model's compiled query SQL. -pub(crate) fn query_sql<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Some(view) = native_model(model)? { - return Ok(view.query_sql(model.py())?.into_any()); - } - native_field(model, "query_sql") -} - -/// The model's `CompiledObjectKey`. -pub(crate) fn model_key<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Some(view) = native_model(model)? { - return Ok(view.key(model.py())?.into_bound(model.py())); - } - native_field(model, "key") -} - -/// The model's destination `CompiledRelationLocation`. -pub(crate) fn destination<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Some(view) = native_model(model)? { - return Ok(view.destination(model.py())?.into_bound(model.py())); - } - native_field(model, "destination") -} - -/// The model's name as a Python string. -pub(crate) fn name_object<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Some(view) = native_model(model)? { - let name: String = view.name(model.py())?; - return Ok(pyo3::types::PyString::new(model.py(), &name).into_any()); - } - native_field(model, "name") -} - -/// The model's attached `SchemaModelEntry`, or `None`. -pub(crate) fn schema_entry<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - if let Some(view) = native_model(model)? { - return Ok(view.schema_entry(py)?.into_bound(py)); - } - native_field(model, "schema_entry") -} - -/// The model's `CompileModelConfig`. -pub(crate) fn config<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - if let Some(view) = native_model(model)? { - return Ok(view.config(py)?.into_bound(py)); - } - native_field(model, "config") -} - -/// The model's `CompileSqlReference` tuple. -pub(crate) fn references<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - if let Some(view) = native_model(model)? { - return Ok(view.references(py)?.into_bound(py).into_any()); - } - native_field(model, "references") -} - -/// The model file's relative path. -pub(crate) fn relative_path<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - if let Some(view) = native_model(model)? { - return Ok(view.relative_path(py)?.into_bound(py)); - } - native_field(model, "relative_path") -} - -/// The model's `InferredColumn` tuple, or `None`. -pub(crate) fn inferred_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - if let Some(view) = native_model(model)? { - return Ok(view.inferred_columns(py)?.into_bound(py)); - } - native_field(model, "inferred_columns") -} - -/// `(resource type, name)` of each dependency. -pub(crate) fn dep_rows(model: &Bound<'_, PyAny>) -> PyResult> { - if let Some(view) = native_model(model)? { - return view.dep_rows(model.py()); - } - native_field(model, "dep_rows")?.extract() -} - -/// The model's dependency `CompiledObjectKey` tuple. -pub(crate) fn deps<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - if let Some(view) = native_model(model)? { - return Ok(view.deps(py)?.into_bound(py).into_any()); - } - native_field(model, "deps") -} - -/// The model's enum columns by name. -pub(crate) fn enum_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - if let Some(view) = native_model(model)? { - return Ok(view.enum_columns(py)?.into_bound(py).into_any()); - } - native_field(model, "enum_columns") -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_host.rs deleted file mode 100644 index 046ad1d0f2..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_host.rs +++ /dev/null @@ -1,122 +0,0 @@ -//! The Python host's slots beside the compiled project's records. - -use std::fmt::{Debug, Formatter, Result as FmtResult}; - -use sqlbuild_project::types::ProjectHost; - -use crate::bindings::_helpers::compiled_views::audit_views::{AuditHandles, AuditProjections}; -use crate::bindings::_helpers::compiled_views::compiled_model_analysis::AnalysisProjections; -use crate::bindings::_helpers::compiled_views::compiled_model_view::CompiledProjections; -use crate::bindings::_helpers::compiled_views::function_views::{ - FunctionHandles, FunctionProjections, -}; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - ModelInputHandles, SchemaHandles, -}; -use crate::bindings::_helpers::compiled_views::seed_views::SeedHandles; -use crate::bindings::_helpers::compiled_views::source_views::{SourceHandles, SourceProjections}; - -/// The extension's host: Python handles and the views built from them. -pub(crate) struct PyProjectHost; - -impl ProjectHost for PyProjectHost { - type ModelInput = ModelInputHandles; - type Schema = SchemaHandles; - type CompiledModel = CompiledProjections; - type Analysis = AnalysisProjections; - type AttachedSource = SourceHandles; - type Source = SourceProjections; - type AttachedFunction = FunctionHandles; - type Function = FunctionProjections; - type AttachedAudit = AuditHandles; - type Audit = AuditProjections; - type Seed = SeedHandles; -} - -impl Debug for ModelInputHandles { - fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { - formatter - .debug_struct("ModelInputHandles") - .finish_non_exhaustive() - } -} - -impl Debug for SchemaHandles { - fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { - formatter - .debug_struct("SchemaHandles") - .finish_non_exhaustive() - } -} - -impl Debug for CompiledProjections { - fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { - formatter - .debug_struct("CompiledProjections") - .finish_non_exhaustive() - } -} - -impl Debug for AnalysisProjections { - fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { - formatter - .debug_struct("AnalysisProjections") - .finish_non_exhaustive() - } -} - -impl Debug for SourceHandles { - fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { - formatter - .debug_struct("SourceHandles") - .finish_non_exhaustive() - } -} - -impl Debug for SourceProjections { - fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { - formatter - .debug_struct("SourceProjections") - .finish_non_exhaustive() - } -} - -impl Debug for FunctionHandles { - fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { - formatter - .debug_struct("FunctionHandles") - .finish_non_exhaustive() - } -} - -impl Debug for FunctionProjections { - fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { - formatter - .debug_struct("FunctionProjections") - .finish_non_exhaustive() - } -} - -impl Debug for AuditHandles { - fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { - formatter - .debug_struct("AuditHandles") - .finish_non_exhaustive() - } -} - -impl Debug for AuditProjections { - fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { - formatter - .debug_struct("AuditProjections") - .finish_non_exhaustive() - } -} - -impl Debug for SeedHandles { - fn fmt(&self, formatter: &mut Formatter<'_>) -> FmtResult { - formatter - .debug_struct("SeedHandles") - .finish_non_exhaustive() - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs deleted file mode 100644 index b3f0994076..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs +++ /dev/null @@ -1,93 +0,0 @@ -//! A compiled project's resource collections, read from its native view. - -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; - -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; -use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -const MODELS_FIELD: &str = "models"; -const SOURCES_FIELD: &str = "sources"; -const SEEDS_FIELD: &str = "seeds"; -const FUNCTIONS_FIELD: &str = "functions"; -const AUDITS_FIELD: &str = "audits"; - -/// The native view behind a project object; any other object is a Python-built record, counted. -#[inline] -fn native_project<'a>(project: &'a Bound<'_, PyAny>) -> PyResult> { - if let Ok(view) = project.cast::() { - return Ok(Some(view.get())); - } - count_stage(project.py(), COMPILED_VIEW_FALLBACKS, 1)?; - Ok(None) -} - -/// The project's compiled models, in compile order. -pub(crate) fn models<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project.py(); - if let Some(view) = native_project(project)? { - return Ok(view.models(py).into_bound(py).into_any()); - } - native_field(project, MODELS_FIELD) -} - -/// The project's compiled sources. -pub(crate) fn sources<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project.py(); - if let Some(view) = native_project(project)? { - return Ok(view.sources(py).into_bound(py)); - } - native_field(project, SOURCES_FIELD) -} - -/// The project's compiled functions. -pub(crate) fn functions<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project.py(); - if let Some(view) = native_project(project)? { - return Ok(view.functions(py).into_bound(py)); - } - native_field(project, FUNCTIONS_FIELD) -} - -/// The project's compiled seeds. -pub(crate) fn seeds<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project.py(); - if let Some(view) = native_project(project)? { - return Ok(view.seeds(py).into_bound(py)); - } - native_field(project, SEEDS_FIELD) -} - -/// The project's compiled audits. -pub(crate) fn audits<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project.py(); - if let Some(view) = native_project(project)? { - return Ok(view.audits(py).into_bound(py)); - } - native_field(project, AUDITS_FIELD) -} - -/// The project's resource collection named `field`; any other field is read by attribute. -pub(crate) fn collection<'py>( - project: &Bound<'py, PyAny>, - field: &str, -) -> PyResult> { - match field { - MODELS_FIELD => models(project), - SOURCES_FIELD => sources(project), - SEEDS_FIELD => seeds(project), - FUNCTIONS_FIELD => functions(project), - AUDITS_FIELD => audits(project), - _ => native_field(project, field), - } -} - -/// The project's compiled SQL tests. -pub(crate) fn sql_tests<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project.py(); - if let Some(view) = native_project(project)? { - return Ok(view.sql_tests(py).into_bound(py)); - } - native_field(project, "sql_tests") -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs deleted file mode 100644 index 841770af33..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs +++ /dev/null @@ -1,612 +0,0 @@ -//! A compiled project, read from the native compiled project that owns its models. - -use std::collections::BTreeMap; -use std::path::PathBuf; - -use pyo3::exceptions::PyTypeError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; -use pyo3::{pyclass, pymethods}; - -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - NativeCompiledModel, model_graph_row, -}; -use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - InputViewClasses, authored_expansion, view_classes, -}; -use crate::bindings::_helpers::compiled_views::seed_views::NativeCompiledSeed; -use crate::bindings::_helpers::compiled_views::source_views::NativeCompiledSource; -use crate::bindings::_helpers::graph::project_graph::{ - NativeProjectGraph, ResourceRow, project_graph, -}; -use sqlbuild_python_base::compile_driver::constants::LINEAGE_GRAPH_BUILDS; -use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -const MODELS_FIELD: &str = "models"; -const RUN_ID_FIELD: &str = "run_id"; -const ENFORCE_EXPLICIT_FIELD: &str = "enforce_explicit_references"; -const KEY_ATTRIBUTE: &str = "key"; -const DEPS_ATTRIBUTE: &str = "deps"; -const TAGS_ATTRIBUTE: &str = "tags"; -const SCHEMA_ENTRY_ATTRIBUTE: &str = "schema_entry"; -const RESOURCE_TYPE_ATTRIBUTE: &str = "resource_type"; -const NAME_ATTRIBUTE: &str = "name"; - -/// Project inputs held as Python objects: user code, discovery or an integration owns them. -struct BoundaryInputs { - /// The resolved connection, adapter integration values. - effective_connection: Py, - /// Project variables, user values. - effective_vars: Py, - /// The native binding catalog. - binding_catalog: Py, - /// The adapter's lexical syntax. - sql_lexical_syntax: Py, - /// The compile cache directory path. - compile_cache_dir: Py, - /// Discovered loader functions, user callables. - loader_functions: Py, - /// Task/asset/check source locations, without retaining additional Python callables. - python_execution_inputs: BTreeMap, - /// Discovered Python hooks, user callables. - hook_functions: Py, - /// Discovered SQL hook files. - sql_hook_files: Py, - /// Discovered custom materializations. - materialization_files: Py, - /// Public enums, user declaration values. - public_enums: Py, - /// Public constants, user declaration values. - public_constants: Py, - /// Loaded macro modules, user code. - loaded_macros: Py, - /// The external reference resolver integration, or `None`. - external_sql_reference_resolver: Py, - /// The native analysis session, or `None`. - native_session: Py, -} - -/// Compiler-derived project parts still held as Python objects until they are owner facts. -struct CompilerHandles { - /// Effective settings. - settings: Py, - /// Effective scenario config. - scenario: Py, - /// Compiled sources. - sources: Py, - /// Compiled seeds. - seeds: Py, - /// Compiled SQL functions. - functions: Py, - /// Compiled audits. - audits: Py, - /// Compiled SQL tests. - sql_tests: Py, - /// Compiled SQL scenarios. - sql_scenarios: Py, - /// Compiler diagnostics. - diagnostics: Py, - /// Literal relations no resource matched. - unmatched_literal_sql_relations: Py, - /// The declaration scope index. - scope_index: Py, -} - -/// The project-level values of one compiled project view. -struct ProjectFields { - run_id: String, - effective_target_name: Option, - effective_target_database: Option, - effective_target_schema: Option, - sql_analysis_dialect: Option, - enforce_explicit_references: bool, - models: Py, - boundary: BoundaryInputs, - compiler: CompilerHandles, -} - -impl ProjectFields { - /// The fields named in `fields`, each required. - fn from_fields(fields: &Bound<'_, PyDict>) -> PyResult { - Ok(Self { - run_id: required(fields, RUN_ID_FIELD)?.extract()?, - effective_target_name: required(fields, "effective_target_name")?.extract()?, - effective_target_database: required(fields, "effective_target_database")?.extract()?, - effective_target_schema: required(fields, "effective_target_schema")?.extract()?, - sql_analysis_dialect: required(fields, "sql_analysis_dialect")?.extract()?, - enforce_explicit_references: required(fields, ENFORCE_EXPLICIT_FIELD)?.extract()?, - models: required(fields, MODELS_FIELD)? - .downcast_into::()? - .unbind(), - boundary: BoundaryInputs { - effective_connection: required(fields, "effective_connection")?.unbind(), - effective_vars: required(fields, "effective_vars")?.unbind(), - binding_catalog: required(fields, "binding_catalog")?.unbind(), - sql_lexical_syntax: required(fields, "sql_lexical_syntax")?.unbind(), - compile_cache_dir: required(fields, "compile_cache_dir")?.unbind(), - loader_functions: required(fields, "loader_functions")?.unbind(), - python_execution_inputs: required(fields, "python_execution_inputs")?.extract()?, - hook_functions: required(fields, "hook_functions")?.unbind(), - sql_hook_files: required(fields, "sql_hook_files")?.unbind(), - materialization_files: required(fields, "materialization_files")?.unbind(), - public_enums: required(fields, "public_enums")?.unbind(), - public_constants: required(fields, "public_constants")?.unbind(), - loaded_macros: required(fields, "loaded_macros")?.unbind(), - external_sql_reference_resolver: required( - fields, - "external_sql_reference_resolver", - )? - .unbind(), - native_session: required(fields, "native_session")?.unbind(), - }, - compiler: CompilerHandles { - settings: required(fields, "settings")?.unbind(), - scenario: required(fields, "scenario")?.unbind(), - sources: required(fields, "sources")?.unbind(), - seeds: required(fields, "seeds")?.unbind(), - functions: required(fields, "functions")?.unbind(), - audits: required(fields, "audits")?.unbind(), - sql_tests: required(fields, "sql_tests")?.unbind(), - sql_scenarios: required(fields, "sql_scenarios")?.unbind(), - diagnostics: required(fields, "diagnostics")?.unbind(), - unmatched_literal_sql_relations: required( - fields, - "unmatched_literal_sql_relations", - )? - .unbind(), - scope_index: required(fields, "scope_index")?.unbind(), - }, - }) - } - - /// These fields with `changes` applied. - fn changed(&self, py: Python<'_>, changes: &Bound<'_, PyDict>) -> PyResult { - let current: Bound<'_, PyDict> = self.as_fields(py)?; - for (name, value) in changes.iter() { - if !current.contains(&name)? { - return Err(PyTypeError::new_err(format!( - "unknown project field {name}" - ))); - } - current.set_item(name, value)?; - } - Self::from_fields(¤t) - } - - /// Every field by name. - fn as_fields<'py>(&self, py: Python<'py>) -> PyResult> { - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item(RUN_ID_FIELD, &self.run_id)?; - fields.set_item("effective_target_name", &self.effective_target_name)?; - fields.set_item("effective_target_database", &self.effective_target_database)?; - fields.set_item("effective_target_schema", &self.effective_target_schema)?; - fields.set_item("sql_analysis_dialect", &self.sql_analysis_dialect)?; - fields.set_item(ENFORCE_EXPLICIT_FIELD, self.enforce_explicit_references)?; - fields.set_item(MODELS_FIELD, self.models.bind(py))?; - fields.set_item( - "effective_connection", - self.boundary.effective_connection.bind(py), - )?; - fields.set_item("effective_vars", self.boundary.effective_vars.bind(py))?; - fields.set_item("binding_catalog", self.boundary.binding_catalog.bind(py))?; - fields.set_item( - "sql_lexical_syntax", - self.boundary.sql_lexical_syntax.bind(py), - )?; - fields.set_item( - "compile_cache_dir", - self.boundary.compile_cache_dir.bind(py), - )?; - fields.set_item("loader_functions", self.boundary.loader_functions.bind(py))?; - fields.set_item( - "python_execution_inputs", - &self.boundary.python_execution_inputs, - )?; - fields.set_item("hook_functions", self.boundary.hook_functions.bind(py))?; - fields.set_item("sql_hook_files", self.boundary.sql_hook_files.bind(py))?; - fields.set_item( - "materialization_files", - self.boundary.materialization_files.bind(py), - )?; - fields.set_item("public_enums", self.boundary.public_enums.bind(py))?; - fields.set_item("public_constants", self.boundary.public_constants.bind(py))?; - fields.set_item("loaded_macros", self.boundary.loaded_macros.bind(py))?; - fields.set_item( - "external_sql_reference_resolver", - self.boundary.external_sql_reference_resolver.bind(py), - )?; - fields.set_item("native_session", self.boundary.native_session.bind(py))?; - fields.set_item("settings", self.compiler.settings.bind(py))?; - fields.set_item("scenario", self.compiler.scenario.bind(py))?; - fields.set_item("sources", self.compiler.sources.bind(py))?; - fields.set_item("seeds", self.compiler.seeds.bind(py))?; - fields.set_item("functions", self.compiler.functions.bind(py))?; - fields.set_item("audits", self.compiler.audits.bind(py))?; - fields.set_item("sql_tests", self.compiler.sql_tests.bind(py))?; - fields.set_item("sql_scenarios", self.compiler.sql_scenarios.bind(py))?; - fields.set_item("diagnostics", self.compiler.diagnostics.bind(py))?; - fields.set_item( - "unmatched_literal_sql_relations", - self.compiler.unmatched_literal_sql_relations.bind(py), - )?; - fields.set_item("scope_index", self.compiler.scope_index.bind(py))?; - Ok(fields) - } -} - -fn required<'py>(fields: &Bound<'py, PyDict>, name: &str) -> PyResult> { - fields - .get_item(name)? - .ok_or_else(|| PyTypeError::new_err(format!("missing project field {name}"))) -} - -/// One compiled project, its models projected from the owner. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeProjectView { - owner: Py, - fields: ProjectFields, - lineage_graph: PyOnceLock>, - sql_expansions: PyOnceLock>, -} - -impl NativeProjectView { - /// The view over `owner` with these project fields. - pub(crate) fn new( - owner: Py, - fields: &Bound<'_, PyDict>, - ) -> PyResult { - Ok(Self::with_fields( - owner, - ProjectFields::from_fields(fields)?, - )) - } - - /// The project field custom-rule facts and rules runs read by `name`, or `None` for any other. - pub(crate) fn fact_field(&self, py: Python<'_>, name: &str) -> Option> { - let field: &Py = match name { - "settings" => &self.fields.compiler.settings, - "sql_scenarios" => &self.fields.compiler.sql_scenarios, - "scope_index" => &self.fields.compiler.scope_index, - "diagnostics" => &self.fields.compiler.diagnostics, - "sources" => &self.fields.compiler.sources, - "seeds" => &self.fields.compiler.seeds, - "functions" => &self.fields.compiler.functions, - "audits" => &self.fields.compiler.audits, - "sql_tests" => &self.fields.compiler.sql_tests, - "public_enums" => &self.fields.boundary.public_enums, - "public_constants" => &self.fields.boundary.public_constants, - "models" => return Some(self.fields.models.clone_ref(py).into_any()), - _ => return None, - }; - Some(field.clone_ref(py)) - } - - /// The compiled project this view projects. - pub(crate) fn owner(&self) -> &Py { - &self.owner - } - - fn with_fields(owner: Py, fields: ProjectFields) -> Self { - Self { - owner, - fields, - lineage_graph: PyOnceLock::new(), - sql_expansions: PyOnceLock::new(), - } - } -} - -#[pymethods] -impl NativeProjectView { - /// This project with `changes` applied; the owner's models are shared. - #[pyo3(signature = (**changes))] - fn with_changes(&self, py: Python<'_>, changes: Option>) -> PyResult { - let fields: ProjectFields = match changes { - Some(changes) => self.fields.changed(py, &changes)?, - None => self.fields.changed(py, &PyDict::new(py))?, - }; - Ok(Self::with_fields(self.owner.clone_ref(py), fields)) - } - - /// Every field by name, for an explicit `CompiledProject` projection. - #[pyo3(signature = (include_sql_expansions = true))] - pub(crate) fn fields<'py>( - &self, - py: Python<'py>, - include_sql_expansions: bool, - ) -> PyResult> { - let fields: Bound<'py, PyDict> = self.fields.as_fields(py)?; - fields.set_item( - "sql_expansions", - if include_sql_expansions { - self.sql_expansions(py)? - } else { - PyDict::new(py).unbind() - }, - )?; - fields.set_item("native_project", self.owner.bind(py))?; - Ok(fields) - } - - #[getter] - pub(crate) fn run_id(&self) -> String { - self.fields.run_id.clone() - } - - #[getter] - pub(crate) fn effective_target_name(&self) -> Option { - self.fields.effective_target_name.clone() - } - - #[getter] - pub(crate) fn effective_target_database(&self) -> Option { - self.fields.effective_target_database.clone() - } - - #[getter] - pub(crate) fn effective_target_schema(&self) -> Option { - self.fields.effective_target_schema.clone() - } - - #[getter] - pub(crate) fn sql_analysis_dialect(&self) -> Option { - self.fields.sql_analysis_dialect.clone() - } - - #[getter(enforce_explicit_references)] - pub(crate) fn is_explicit_reference_enforced(&self) -> bool { - self.fields.enforce_explicit_references - } - - #[getter] - pub(crate) fn models(&self, py: Python<'_>) -> Py { - self.fields.models.clone_ref(py) - } - - #[getter] - pub(crate) fn native_project(&self, py: Python<'_>) -> Py { - self.owner.clone_ref(py) - } - - #[getter] - pub(crate) fn effective_connection(&self, py: Python<'_>) -> Py { - self.fields.boundary.effective_connection.clone_ref(py) - } - - #[getter] - pub(crate) fn effective_vars(&self, py: Python<'_>) -> Py { - self.fields.boundary.effective_vars.clone_ref(py) - } - - #[getter] - pub(crate) fn binding_catalog(&self, py: Python<'_>) -> Py { - self.fields.boundary.binding_catalog.clone_ref(py) - } - - #[getter] - pub(crate) fn sql_lexical_syntax(&self, py: Python<'_>) -> Py { - self.fields.boundary.sql_lexical_syntax.clone_ref(py) - } - - #[getter] - pub(crate) fn compile_cache_dir(&self, py: Python<'_>) -> Py { - self.fields.boundary.compile_cache_dir.clone_ref(py) - } - - #[getter] - pub(crate) fn loader_functions(&self, py: Python<'_>) -> Py { - self.fields.boundary.loader_functions.clone_ref(py) - } - - #[getter] - pub(crate) fn python_execution_inputs(&self) -> BTreeMap { - self.fields.boundary.python_execution_inputs.clone() - } - - #[getter] - pub(crate) fn hook_functions(&self, py: Python<'_>) -> Py { - self.fields.boundary.hook_functions.clone_ref(py) - } - - #[getter] - pub(crate) fn sql_hook_files(&self, py: Python<'_>) -> Py { - self.fields.boundary.sql_hook_files.clone_ref(py) - } - - #[getter] - pub(crate) fn materialization_files(&self, py: Python<'_>) -> Py { - self.fields.boundary.materialization_files.clone_ref(py) - } - - #[getter] - pub(crate) fn public_enums(&self, py: Python<'_>) -> Py { - self.fields.boundary.public_enums.clone_ref(py) - } - - #[getter] - pub(crate) fn public_constants(&self, py: Python<'_>) -> Py { - self.fields.boundary.public_constants.clone_ref(py) - } - - #[getter] - pub(crate) fn loaded_macros(&self, py: Python<'_>) -> Py { - self.fields.boundary.loaded_macros.clone_ref(py) - } - - #[getter] - pub(crate) fn external_sql_reference_resolver(&self, py: Python<'_>) -> Py { - self.fields - .boundary - .external_sql_reference_resolver - .clone_ref(py) - } - - #[getter] - pub(crate) fn native_session(&self, py: Python<'_>) -> Py { - self.fields.boundary.native_session.clone_ref(py) - } - - #[getter] - pub(crate) fn settings(&self, py: Python<'_>) -> Py { - self.fields.compiler.settings.clone_ref(py) - } - - #[getter] - pub(crate) fn scenario(&self, py: Python<'_>) -> Py { - self.fields.compiler.scenario.clone_ref(py) - } - - #[getter] - pub(crate) fn sources(&self, py: Python<'_>) -> Py { - self.fields.compiler.sources.clone_ref(py) - } - - #[getter] - pub(crate) fn seeds(&self, py: Python<'_>) -> Py { - self.fields.compiler.seeds.clone_ref(py) - } - - #[getter] - pub(crate) fn functions(&self, py: Python<'_>) -> Py { - self.fields.compiler.functions.clone_ref(py) - } - - #[getter] - pub(crate) fn audits(&self, py: Python<'_>) -> Py { - self.fields.compiler.audits.clone_ref(py) - } - - #[getter] - pub(crate) fn sql_tests(&self, py: Python<'_>) -> Py { - self.fields.compiler.sql_tests.clone_ref(py) - } - - #[getter] - pub(crate) fn sql_scenarios(&self, py: Python<'_>) -> Py { - self.fields.compiler.sql_scenarios.clone_ref(py) - } - - #[getter] - pub(crate) fn diagnostics(&self, py: Python<'_>) -> Py { - self.fields.compiler.diagnostics.clone_ref(py) - } - - #[getter] - pub(crate) fn unmatched_literal_sql_relations(&self, py: Python<'_>) -> Py { - self.fields - .compiler - .unmatched_literal_sql_relations - .clone_ref(py) - } - - #[getter] - pub(crate) fn scope_index(&self, py: Python<'_>) -> Py { - self.fields.compiler.scope_index.clone_ref(py) - } - - /// Each model's authored-to-expanded passes by file path, where interpolation kept the query. - #[getter] - pub(crate) fn sql_expansions(&self, py: Python<'_>) -> PyResult> { - self.sql_expansions - .get_or_try_init(py, || { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - let owner = self.owner.bind(py).borrow(); - let expansions: Bound<'_, PyDict> = PyDict::new(py); - for record in owner.inputs() { - let expansion: Py = authored_expansion(&classes, record)?; - if !expansion.is_none(py) { - let path: Bound<'_, PyAny> = - record.host.model_file.get().path_objects(py)?.0; - expansions.set_item(path, expansion)?; - } - } - Ok::<_, pyo3::PyErr>(expansions.unbind()) - }) - .map(|expansions| expansions.clone_ref(py)) - } - - /// The lineage edges, tags, model folders and names, indexed natively once per project. - #[getter] - pub(crate) fn lineage_graph(&self, py: Python<'_>) -> PyResult> { - self.lineage_graph - .get_or_try_init(py, || { - let mut rows: Vec = Vec::new(); - for model in self.fields.models.bind(py).iter() { - let model: Bound<'_, NativeCompiledModel> = model.downcast_into()?; - rows.push(model_graph_row(model.get(), py)?); - } - let compiler: &CompilerHandles = &self.fields.compiler; - for source in compiler.sources.bind(py).try_iter()? { - let source: Bound<'_, PyAny> = source?; - rows.push(match source.cast::() { - Ok(native) => native.get().graph_row(py)?, - Err(_) => resource_row(&source, None)?, - }); - } - for seed in compiler.seeds.bind(py).try_iter()? { - let seed: Bound<'_, PyAny> = seed?; - rows.push(match seed.cast::() { - Ok(native) => native.get().graph_row(py)?, - Err(_) => { - let tags: Bound<'_, PyAny> = native_field( - &native_field(&seed, SCHEMA_ENTRY_ATTRIBUTE)?, - TAGS_ATTRIBUTE, - )?; - resource_row(&seed, Some(&tags))? - } - }); - } - for function in compiler.functions.bind(py).try_iter()? { - let function: Bound<'_, PyAny> = function?; - rows.push(match function.cast::() { - Ok(native) => native.get().graph_row(py)?, - Err(_) => { - let tags: Bound<'_, PyAny> = native_field(&function, TAGS_ATTRIBUTE)?; - resource_row(&function, Some(&tags))? - } - }); - } - self.owner.bind(py).borrow_mut().record_graph_build(); - count_stage(py, LINEAGE_GRAPH_BUILDS, 1)?; - Py::new(py, project_graph(rows)?) - }) - .map(|graph| graph.clone_ref(py)) - } -} - -/// A source, seed or function's graph row, read from its compiled object. -fn resource_row( - resource: &Bound<'_, PyAny>, - tags: Option<&Bound<'_, PyAny>>, -) -> PyResult { - let mut deps: Vec<(String, String)> = Vec::new(); - for key in native_field(resource, DEPS_ATTRIBUTE)?.try_iter()? { - deps.push(key_row(&key?)?); - } - let mut tag_names: Vec = Vec::new(); - if let Some(tags) = tags { - for tag in tags.try_iter()? { - tag_names.push(tag?.str()?.extract()?); - } - } - Ok(ResourceRow { - key: key_row(&native_field(resource, KEY_ATTRIBUTE)?)?, - deps, - tags: tag_names, - folder: None, - }) -} - -/// `(resource type, name)` of a `CompiledObjectKey`. -pub(crate) fn key_row(key: &Bound<'_, PyAny>) -> PyResult<(String, String)> { - Ok(( - native_field(key, RESOURCE_TYPE_ATTRIBUTE)? - .str()? - .extract()?, - native_field(key, NAME_ATTRIBUTE)?.extract()?, - )) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/records.rs deleted file mode 100644 index 17ecfeb899..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/records.rs +++ /dev/null @@ -1,559 +0,0 @@ -//! Compiled records and views: projecting native views as records, and recording later stages. - -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; -use pyo3::types::{ - PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyList, PyTuple, PyType, -}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; - -use crate::bindings::_helpers::compiled_project::project::{ - AnalysisPy, NativeCompiledProject, ProofPy, model_facts, -}; -use crate::bindings::_helpers::compiled_project::schema_facts::entry_schema_fact; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::inputs_view::NativeInputsView; -use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::record_spec::record_spec; -use sqlbuild_python_base::records::types::RecordId; - -const DATACLASSES_MODULE: &str = "dataclasses"; -const DECIMAL_KIND: &str = "decimal"; -const COLLECTION_KINDS: [&str; 2] = ["list", "set"]; -const SCALAR_KINDS: [&str; 5] = ["string", "integer", "boolean", "float", "null"]; - -/// `dataclasses.replace(value, **changes)`. -pub(crate) fn replace<'py>( - value: &Bound<'py, PyAny>, - changes: &[(&str, Bound<'py, PyAny>)], -) -> PyResult> { - let py: Python<'py> = value.py(); - let keywords: Bound<'py, PyDict> = PyDict::new(py); - for (name, change) in changes { - keywords.set_item(*name, change)?; - } - imported(py, DATACLASSES_MODULE, "replace")?.call((value,), Some(&keywords)) -} - -/// `value` when it already is a `class` record, else `class(**{field: getattr(value, field)})`. -fn record<'py>(value: &Bound<'py, PyAny>, class: RecordId) -> PyResult> { - let py: Python<'py> = value.py(); - let class: Bound<'py, PyAny> = record_class(py, class)?; - if value.is_instance(&class)? { - return Ok(value.clone()); - } - let fields: Bound<'py, PyDict> = PyDict::new(py); - for field in record_spec(class.cast::()?)?.fields { - fields.set_item(field.name, native_field(value, field.name)?)?; - } - class.call((), Some(&fields)) -} - -/// The model as a `CompiledModel`, projected from its owner when it is a native view. -#[pyfunction] -pub(crate) fn compiled_model_record<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - record(model, RecordId::CompiledModel) -} - -/// The audit as a `CompiledAudit`, projected from its owner when it is a native view. -#[pyfunction] -pub(crate) fn compiled_audit_record<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { - record(audit, RecordId::CompiledAudit) -} - -/// The function as a `CompiledFunction`, projected from its owner when it is a native view. -#[pyfunction] -pub(crate) fn compiled_function_record<'py>( - function: &Bound<'py, PyAny>, -) -> PyResult> { - record(function, RecordId::CompiledFunction) -} - -/// The source as a `CompiledSource`, projected from its owner when it is a native view. -#[pyfunction] -pub(crate) fn compiled_source_record<'py>( - source: &Bound<'py, PyAny>, -) -> PyResult> { - record(source, RecordId::CompiledSource) -} - -/// The seed as a `CompiledSeed`, projected from its owner when it is a native view. -#[pyfunction] -pub(crate) fn compiled_seed_record<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - record(seed, RecordId::CompiledSeed) -} - -/// Each item of `fields[name]` as a `class` record. -fn records_of<'py>(fields: &Bound<'py, PyDict>, (name, class): (&str, RecordId)) -> PyResult<()> { - let py: Python<'py> = fields.py(); - let items: Vec> = fields - .as_any() - .get_item(name)? - .try_iter()? - .map(|item| record(&item?, class)) - .collect::>()?; - fields.set_item(name, PyTuple::new(py, items)?) -} - -/// The project as a `CompiledProject`, projected from its owner when it is a native view. -#[pyfunction] -#[pyo3(signature = (project, *, include_sql_expansions=true))] -pub(crate) fn compiled_project_record<'py>( - project: &Bound<'py, PyAny>, - include_sql_expansions: bool, -) -> PyResult> { - let py: Python<'py> = project.py(); - let class: Bound<'py, PyAny> = record_class(py, RecordId::CompiledProject)?; - if project.is_instance(&class)? { - if include_sql_expansions { - return Ok(project.clone()); - } - return replace(project, &[("sql_expansions", PyDict::new(py).into_any())]); - } - let view: PyRef<'py, NativeProjectView> = project.extract()?; - let fields: Bound<'py, PyDict> = view.fields(py, include_sql_expansions)?; - drop(view); - for pair in [ - ("seeds", RecordId::CompiledSeed), - ("sources", RecordId::CompiledSource), - ("functions", RecordId::CompiledFunction), - ("audits", RecordId::CompiledAudit), - ] { - records_of(&fields, pair)?; - } - new_record(py, RecordId::CompiledProject, (), Some(&fields)) -} - -/// The model with these final binding diagnostics. -#[pyfunction] -#[pyo3(signature = (*, model, diagnostics))] -pub(crate) fn with_binding_diagnostics<'py>( - model: &Bound<'py, PyAny>, - diagnostics: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = model.py(); - if model.is_instance(&record_class(py, RecordId::CompiledModel)?)? { - return replace(model, &[("binding_diagnostics", diagnostics.clone())]); - } - model.call_method1("with_binding_diagnostics", (diagnostics,)) -} - -/// The model with its recovered bindings and its poisoned outputs' types forgotten. -#[pyfunction] -#[pyo3(signature = (*, model, diagnostics, poisoned))] -pub(crate) fn with_type_recovery<'py>( - model: &Bound<'py, PyAny>, - diagnostics: &Bound<'py, PyAny>, - poisoned: &Bound<'py, PyFrozenSet>, -) -> PyResult> { - let py: Python<'py> = model.py(); - let name: String = native_field(model, "name")?.extract()?; - let mut unchecked: Vec = Vec::new(); - for pair in poisoned.iter() { - let (model_name, column): (String, String) = pair.extract()?; - if model_name == name { - unchecked.push(column); - } - } - if !model.is_instance(&record_class(py, RecordId::CompiledModel)?)? { - return model.call_method1("with_type_recovery", (diagnostics, unchecked)); - } - let inferred: Bound<'py, PyAny> = native_field(model, "inferred_columns")?; - let columns: Bound<'py, PyAny> = if inferred.is_none() { - inferred - } else { - let mut columns: Vec> = Vec::new(); - for column in inferred.try_iter()? { - let column: Bound<'py, PyAny> = column?; - let key = (name.as_str(), native_field(&column, "name")?); - columns.push(if poisoned.contains(key)? { - replace(&column, &[("type", py.None().into_bound(py))])? - } else { - column - }); - } - PyTuple::new(py, columns)?.into_any() - }; - replace( - model, - &[ - ("binding_diagnostics", diagnostics.clone()), - ("inferred_columns", columns), - ( - "unchecked_output_columns", - PyFrozenSet::new(py, &unchecked)?.into_any(), - ), - ], - ) -} - -/// The project with `changes`; a native view shares its owner's models. -#[pyfunction] -#[pyo3(signature = (*, project, **changes))] -pub(crate) fn project_with<'py>( - project: &Bound<'py, PyAny>, - changes: Option>, -) -> PyResult> { - with_changes(project, RecordId::CompiledProject, changes) -} - -/// The compile inputs with `changes`; a native view shares its owner's model inputs. -#[pyfunction] -#[pyo3(signature = (*, inputs, **changes))] -pub(crate) fn inputs_with<'py>( - inputs: &Bound<'py, PyAny>, - changes: Option>, -) -> PyResult> { - with_changes(inputs, RecordId::CompileProjectInputs, changes) -} - -/// `replace(value, **changes)` for a record, else the native view's `with_changes(**changes)`. -fn with_changes<'py>( - value: &Bound<'py, PyAny>, - class: RecordId, - changes: Option>, -) -> PyResult> { - let py: Python<'py> = value.py(); - let changes: Bound<'py, PyDict> = changes.unwrap_or_else(|| PyDict::new(py)); - if value.is_instance(&record_class(py, class)?)? { - return imported(py, DATACLASSES_MODULE, "replace")?.call((value,), Some(&changes)); - } - value.call_method("with_changes", (), Some(&changes)) -} - -/// The native owner's inputs view when it holds every model input; else a record. -#[pyfunction] -#[pyo3(signature = (*, fields, native_project))] -pub(crate) fn compile_inputs_view<'py>( - fields: &Bound<'py, PyDict>, - native_project: Option>, -) -> PyResult> { - let py: Python<'py> = fields.py(); - if let Some(project) = &native_project { - let mut native: bool = true; - for model in fields.as_any().get_item("model_inputs")?.try_iter()? { - if !model?.is_instance_of::() { - native = false; - break; - } - } - if native { - let view: NativeInputsView = - NativeCompiledProject::inputs_view(project.clone(), fields.clone())?; - return view.into_bound_py_any(py); - } - } - let keywords: Bound<'py, PyDict> = fields.copy()?; - keywords.set_item("native_project", native_project)?; - new_record(py, RecordId::CompileProjectInputs, (), Some(&keywords)) -} - -/// A typed SQL value as the plain value native consumers read. -#[pyfunction] -pub(crate) fn plain_typed_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = value.py(); - let kind: String = native_field(value, "kind")?.getattr("value")?.extract()?; - let inner: Bound<'py, PyAny> = native_field(value, "value")?; - if kind == DECIMAL_KIND { - return Ok(inner.str()?.into_any()); - } - if SCALAR_KINDS.contains(&kind.as_str()) { - return Ok(inner); - } - if COLLECTION_KINDS.contains(&kind.as_str()) { - let items: Vec> = inner - .try_iter()? - .map(|item| plain_typed_value(&item?)) - .collect::>()?; - return Ok(PyList::new(py, items)?.into_any()); - } - let mapping: Bound<'py, PyDict> = PyDict::new(py); - for pair in inner.try_iter()? { - let (key, item): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; - mapping.set_item(key, plain_typed_value(&item)?)?; - } - Ok(mapping.into_any()) -} - -/// An enum declaration as the native project row. -fn enum_row<'py>(declaration: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = declaration.py(); - let members: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = - native_field(declaration, "members")? - .try_iter()? - .map(|member| { - let member: Bound<'py, PyAny> = member?; - Ok(( - native_field(&member, "name")?, - native_field(&member, "value")?, - )) - }) - .collect::>()?; - PyTuple::new( - py, - [ - native_field(declaration, "name")?, - native_field(declaration, "relative_path")?.call_method0("as_posix")?, - members.into_bound_py_any(py)?, - py.None().into_bound(py), - py.None().into_bound(py), - py.None().into_bound(py), - ], - ) -} - -/// A constant declaration as the native project row, its value made JSON-safe. -fn constant_row<'py>(declaration: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = declaration.py(); - let render_as: Bound<'py, PyAny> = native_field(declaration, "render_as")?; - PyTuple::new( - py, - [ - native_field(declaration, "name")?, - native_field(declaration, "relative_path")?.call_method0("as_posix")?, - PyList::empty(py).into_any(), - plain_typed_value(&native_field(declaration, "value")?)?, - native_field(declaration, "logical_type")?.getattr("display_name")?, - if render_as.is_none() { - render_as - } else { - native_field(&render_as, "value")? - }, - ], - ) -} - -/// Enum and constant declarations as native rows, in the given order. -#[pyfunction] -#[pyo3(signature = (*, enums, constants))] -pub(crate) fn declaration_rows<'py>( - enums: &Bound<'py, PyAny>, - constants: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyList>, Bound<'py, PyList>)> { - let py: Python<'py> = enums.py(); - let enum_rows: Vec> = enums - .try_iter()? - .map(|declaration| enum_row(&declaration?)) - .collect::>()?; - let constant_rows: Vec> = constants - .try_iter()? - .map(|declaration| constant_row(&declaration?)) - .collect::>()?; - Ok((PyList::new(py, enum_rows)?, PyList::new(py, constant_rows)?)) -} - -/// `(name, type)` of each dynamic family; `casefold` keys the family by its folded name. -fn family_types<'py>( - families: &Bound<'py, PyAny>, - casefold: bool, -) -> PyResult)>> { - families - .try_iter()? - .map(|family| { - let family: Bound<'py, PyAny> = family?; - let name: Bound<'py, PyAny> = native_field(&family, "name")?; - let name: String = if casefold { - name.call_method0("casefold")?.extract()? - } else { - name.extract()? - }; - Ok((name, native_field(&family, "inferred_type")?.extract()?)) - }) - .collect() -} - -/// Transfer each assembled model's typed analysis to its retained facts; return how many. -#[pyfunction] -#[pyo3(signature = (*, project, models))] -pub(crate) fn record_model_analyses<'py>( - project: &Bound<'py, NativeCompiledProject>, - models: &Bound<'py, PyAny>, -) -> PyResult { - let py: Python<'py> = project.py(); - let models: Vec> = models.try_iter()?.collect::>()?; - let views: Vec> = models - .iter() - .filter(|model| model.is_instance_of::()) - .map(|model| model.extract::>()) - .collect::>()?; - let mut transferred: usize = NativeCompiledProject::record_view_analyses(project, views)?; - let nullable: Bound<'py, PyAny> = enum_member(py, RecordId::InferredNullability, "NULLABLE")?; - for model in &models { - if model.is_instance_of::() { - continue; - } - let inferred: Bound<'py, PyAny> = native_field(model, "inferred_rows")?; - let rows: Vec<(String, Option, Bound<'py, PyAny>)> = if inferred.is_none() { - Vec::new() - } else { - inferred.extract()? - }; - let proof: Bound<'py, PyAny> = native_field(model, "dynamic_column_contract")?; - let proof = if proof.is_none() { - None - } else { - let families: Bound<'py, PyAny> = native_field(&proof, "families")?; - Some(ProofPy { - output_proven: native_field(&proof, "output_proven")?.extract()?, - bare_dynamic_pivot: native_field(&proof, "bare_dynamic_pivot")?.extract()?, - families: family_types(&families, true)?, - failure_reason: native_field(&proof, "failure_reason")?.extract()?, - named_families: family_types(&families, false)?, - }) - }; - let analysis = AnalysisPy { - inferred: !inferred.is_none(), - inferred_columns: rows - .iter() - .map(|(name, column_type, _)| (name.clone(), column_type.clone())) - .collect(), - nullable_columns: rows - .iter() - .map(|(_, _, nullability)| nullability.eq(&nullable)) - .collect::>()?, - fast_lineage_has_star: native_field(model, "fast_lineage_has_star")?.is_truthy()?, - unchecked: native_field(model, "unchecked_output_columns")? - .try_iter()? - .map(|column| column?.extract()) - .collect::>()?, - }; - let name: String = native_field(model, "name")?.extract()?; - transferred += usize::from( - project - .borrow_mut() - .record_model_analysis(&name, analysis, proof), - ); - } - Ok(transferred) -} - -/// A native project for a `CompiledProject` built without the compile-input stage. -fn hand_built_project<'py>( - project: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project.py(); - let native: Bound<'py, NativeCompiledProject> = - Bound::new(py, NativeCompiledProject::default())?; - let models: Bound<'py, PyAny> = native_field(project, "models")?; - for model in models.try_iter()? { - let model: Bound<'py, PyAny> = model?; - let mut facts: CompiledModelFacts = model_facts(model_row(&model)?.as_any().extract()?)?; - facts.schema = entry_schema_fact( - &native_field(&model, "schema_entry")?, - &native_field(&model, "relative_path")?, - )?; - native.borrow_mut().retain(facts); - } - let _ = record_model_analyses(&native, &models)?; - Ok(native) -} - -/// The compile's retained project, or one recorded from a hand-built project. -#[pyfunction] -pub(crate) fn compiled_project_facts<'py>( - project: &Bound<'py, PyAny>, -) -> PyResult> { - if let Ok(view) = project.downcast::() { - return Ok(view.get().owner().bind(project.py()).clone()); - } - let retained: Bound<'py, PyAny> = native_field(project, "native_project")?; - if !retained.is_none() { - return Ok(retained.downcast_into()?); - } - hand_built_project(project) -} - -fn model_row<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - let relative_path: Bound<'py, PyAny> = native_field(model, "relative_path")?; - let config: Bound<'py, PyAny> = native_field(model, "config")?; - let references: Vec> = native_field(model, "references")? - .try_iter()? - .map(|reference| { - let reference: Bound<'py, PyAny> = reference?; - PyTuple::new( - py, - [ - native_field(&reference, "ref_kind")?.str()?.into_any(), - native_field(&reference, "ref_name")?, - native_field(&reference, "ref_package")?, - ], - ) - }) - .collect::>()?; - let (enums, constants) = declaration_rows( - &native_field(model, "enum_declarations")?, - &native_field(model, "constant_declarations")?, - )?; - PyTuple::new( - py, - [ - PyTuple::new( - py, - [ - native_field(model, "name")?, - relative_path.call_method0("as_posix")?, - native_field(model, "query_sql")?, - native_field(model, "authored_query_sql")?, - native_field(model, "authored_sql")?, - ], - )? - .into_any(), - PyTuple::new( - py, - [ - native_field(&config, "values")?, - PyList::new( - py, - native_field(&config, "model_header_keys")? - .try_iter()? - .collect::>>()?, - )? - .into_any(), - native_field(&config, "layer_schema")?, - ], - )? - .into_any(), - PyList::new(py, references)?.into_any(), - PyList::new( - py, - native_field(model, "enum_columns")? - .try_iter()? - .collect::>>()?, - )? - .into_any(), - PyTuple::new(py, [enums.into_any(), constants.into_any()])?.into_any(), - ], - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - for function in [ - wrap_pyfunction!(compiled_model_record, module)?, - wrap_pyfunction!(compiled_audit_record, module)?, - wrap_pyfunction!(compiled_function_record, module)?, - wrap_pyfunction!(compiled_source_record, module)?, - wrap_pyfunction!(compiled_seed_record, module)?, - wrap_pyfunction!(compiled_project_record, module)?, - wrap_pyfunction!(with_binding_diagnostics, module)?, - wrap_pyfunction!(with_type_recovery, module)?, - wrap_pyfunction!(project_with, module)?, - wrap_pyfunction!(inputs_with, module)?, - wrap_pyfunction!(compile_inputs_view, module)?, - wrap_pyfunction!(plain_typed_value, module)?, - wrap_pyfunction!(declaration_rows, module)?, - wrap_pyfunction!(record_model_analyses, module)?, - wrap_pyfunction!(compiled_project_facts, module)?, - ] { - module.add_function(function)?; - } - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/registration.rs index 348c352052..05c8d30010 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/registration.rs @@ -1,5 +1,6 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; +use sqlbuild_python_project::compiled_views::main::project::register_records::register_records; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - crate::bindings::_helpers::compiled_views::records::register(module) + register_records(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs deleted file mode 100644 index 19829fd9a8..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs +++ /dev/null @@ -1,373 +0,0 @@ -//! Declared unique keys of project relations, derived natively from their key facts. - -use sqlbuild_python_base::positional_record; -use std::collections::BTreeMap; -use std::path::PathBuf; - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; -use sqlbuild_discovery::project_validation::main::at_most_one_csv_data_row::at_most_one_csv_data_row; -use sqlbuild_discovery::project_validation::models::CsvSettings; -use sqlbuild_rules::relation_keys::main::declared_relation_keys::declared_relation_keys as derive_keys; -use sqlbuild_rules::relation_keys::models::{DeclaredKeyColumn, DeclaredRelation}; - -use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - NativeModelInput, model_relation_key_row, -}; -use crate::bindings::_helpers::compiled_views::source_views::{ - SourceKeyFacts, SourceViews, read_source_view, -}; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::discovery::main::project::seed_csv_settings::seed_csv_settings; - -const MATERIALIZED_CONFIG: &str = "materialized"; -const INCREMENTAL_MODE_CONFIG: &str = "incremental_mode"; -const INCREMENTAL_STRATEGY_CONFIG: &str = "incremental_strategy"; -const CURSOR_CONFIG: &str = "cursor"; -const UNIQUE_KEY_CONFIG: &str = "unique_key"; -const UNIQUE_AUDIT_COLUMN_ARGUMENT: &str = "column"; -const RELATION_KIND_SEED: &str = "seed"; -const RELATION_KIND_SOURCE: &str = "source"; - -/// Native declared columns and model audits used to derive relation keys. -type DeclaredKeyFacts<'a> = (&'a [SchemaColumnFact], Vec<(String, Option)>); - -positional_record! { - extract, into; - /// `(kind, name, materialized, mode, strategy, cursor, unique key, columns, audits)`. - pub(crate) struct RelationKeyRow { - kind: String, - name: String, - materialized: String, - mode: String, - strategy: String, - has_cursor: bool, - unique_key: Option>, - columns: Vec<(String, Vec, bool)>, - audits: Vec<(String, Option)>, - } -} - -/// The key row of one relation from its config values and declared column and audit specs. -pub(crate) fn relation_key_row( - kind: &str, - name: String, - values: &Bound<'_, PyDict>, - declared: Option>, -) -> PyResult { - let mut columns: Vec<(String, Vec, bool)> = Vec::new(); - let mut audits: Vec<(String, Option)> = Vec::new(); - if let Some((declared_columns, declared_audits)) = declared { - for column in declared_columns { - columns.push(( - column.name.clone(), - column.audit_definitions.clone(), - column.nullable == Some(false), - )); - } - audits = declared_audits; - } - Ok(RelationKeyRow { - kind: kind.to_owned(), - name, - materialized: config_text(values, MATERIALIZED_CONFIG)?, - mode: config_text(values, INCREMENTAL_MODE_CONFIG)?, - strategy: config_text(values, INCREMENTAL_STRATEGY_CONFIG)?, - has_cursor: values - .get_item(CURSOR_CONFIG)? - .is_some_and(|value| value.is_truthy().unwrap_or(false)), - unique_key: unique_key(values)?, - columns, - audits, - }) -} - -/// Each audit spec's `(definition name, column argument when it is text)`. -pub(crate) fn audit_key_rows(audits: &Bound<'_, PyAny>) -> PyResult)>> { - let mut rows: Vec<(String, Option)> = Vec::new(); - for audit in audits.try_iter()? { - let audit: Bound<'_, PyAny> = audit?; - let argument: Option> = native_field(&audit, "arguments")? - .downcast::()? - .get_item(UNIQUE_AUDIT_COLUMN_ARGUMENT)?; - rows.push(( - native_field(&audit, "definition_name")?.extract()?, - argument - .filter(pyo3::types::PyAnyMethods::is_instance_of::) - .map(|value| value.extract()) - .transpose()?, - )); - } - Ok(rows) -} - -/// A source's key row from its native view's declared facts, or `None` for any other object. -pub(crate) fn source_view_key_row(view: &Bound<'_, PyAny>) -> Option> { - read_source_view(view, |source, py| { - let facts: &SourceKeyFacts = source.key_facts(py)?; - Ok(RelationKeyRow { - kind: RELATION_KIND_SOURCE.to_owned(), - name: source.attached().name.clone(), - materialized: String::new(), - mode: String::new(), - strategy: String::new(), - has_cursor: false, - unique_key: None, - columns: facts - .columns - .iter() - .map(|column| { - ( - column.name.clone(), - column.audit_definitions.clone(), - column.nullable == Some(false), - ) - }) - .collect(), - audits: facts.audits.clone(), - }) - }) -} - -/// What `relation_keys` reads for a source: its native view itself, or its `source_entry`. -pub(crate) fn source_key_input<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - if read_source_view(source, |_, _| Ok(())).is_some() { - return Ok(source.clone()); - } - native_field(source, "source_entry") -} - -/// `str(values.get(key, ""))`. -fn config_text(values: &Bound<'_, PyDict>, key: &str) -> PyResult { - match values.get_item(key)? { - Some(value) => value.str()?.extract(), - None => Ok(String::new()), - } -} - -/// `unique_key` as a name list, when it is a name or a non-empty list or tuple of names. -fn unique_key(values: &Bound<'_, PyDict>) -> PyResult>> { - let Some(value) = values.get_item(UNIQUE_KEY_CONFIG)? else { - return Ok(None); - }; - if value.is_instance_of::() { - return Ok(Some(vec![value.extract()?])); - } - if !(value.is_instance_of::() || value.is_instance_of::()) { - return Ok(None); - } - let mut names: Vec = Vec::new(); - for item in value.try_iter()? { - let item: Bound<'_, PyAny> = item?; - if !item.is_instance_of::() { - return Ok(None); - } - names.push(item.str()?.extract()?); - } - Ok(Some(names)) -} - -/// The key row of a seed, source or hand-built model from its config values and schema entry. -pub(crate) fn entry_relation_key_row( - kind: &str, - name: String, - values: &Bound<'_, PyDict>, - entry: &Bound<'_, PyAny>, -) -> PyResult { - if entry.is_none() { - return relation_key_row(kind, name, values, None); - } - let columns: Vec = - python_column_facts(&declared_specs(entry, "columns")?, None)?; - let audits: Bound<'_, PyAny> = declared_specs(entry, "audits")?; - relation_key_row( - kind, - name, - values, - Some((&columns, audit_key_rows(&audits)?)), - ) -} - -/// `getattr(entry, name, None) or ()`. -fn declared_specs<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult> { - match entry.getattr_opt(name)? { - Some(specs) if specs.is_truthy()? => Ok(specs), - _ => Ok(PyTuple::empty(entry.py()).into_any()), - } -} - -/// Declared keys by `kind:name` identity, in identity order. -pub(crate) type RelationKeys = Vec<(String, Vec>)>; - -/// Names of the seeds whose CSV holds at most one data row, from `seed_file` and `schema_entry`. -pub(crate) fn one_row_seed_names(seeds: &[Bound<'_, PyAny>]) -> PyResult> { - let mut candidates: Vec<(String, PathBuf, CsvSettings)> = Vec::with_capacity(seeds.len()); - for seed in seeds { - let entry: Bound<'_, PyAny> = native_field(seed, "schema_entry")?; - candidates.push(( - native_field(&entry, "name")?.extract()?, - native_field(&native_field(seed, "seed_file")?, "file_path")?.extract()?, - seed_csv_settings(&entry)?, - )); - } - let py: Python<'_> = match seeds.first() { - Some(seed) => seed.py(), - None => return Ok(Vec::new()), - }; - Ok(py.detach(|| { - candidates - .into_iter() - .filter(|(_, path, settings)| at_most_one_csv_data_row(path, settings)) - .map(|(name, _, _)| name) - .collect() - })) -} - -/// Keys proving one row per key by `kind:name`; seeds in `one_row_seeds` also get the empty key. -pub(crate) fn relation_keys<'py>( - py: Python<'py>, - mut rows: Vec, - (seeds, sources): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - one_row_seeds: &[String], -) -> PyResult { - for (kind, entries) in [(RELATION_KIND_SEED, seeds), (RELATION_KIND_SOURCE, sources)] { - for entry in entries.try_iter()? { - let entry: Bound<'py, PyAny> = entry?; - if let Some(row) = source_view_key_row(&entry) { - rows.push(row?); - continue; - } - let name: String = native_field(&entry, "name")?.extract()?; - rows.push(entry_relation_key_row( - kind, - name, - &PyDict::new(py), - &entry, - )?); - } - } - let mut relations: Vec = rows.into_iter().map(declared_relation).collect(); - for relation in &mut relations { - relation.at_most_one_row = - relation.kind == RELATION_KIND_SEED && one_row_seeds.contains(&relation.name); - } - let keys: BTreeMap>> = py.detach(|| derive_keys(&relations)); - Ok(keys.into_iter().collect()) -} - -/// `keys` as a `{identity: ((column, ...), ...)}` dict. -fn keys_dict<'py>(py: Python<'py>, keys: RelationKeys) -> PyResult> { - let found: Bound<'py, PyDict> = PyDict::new(py); - for (identity, unique_keys) in keys { - let mut tuples: Vec> = Vec::with_capacity(unique_keys.len()); - for key in unique_keys { - tuples.push(PyTuple::new(py, key)?); - } - found.set_item(identity, PyTuple::new(py, tuples)?)?; - } - Ok(found) -} - -/// Keys that prove one row per key, by `kind:name` identity, from model rows, seeds and sources. -#[pyfunction] -#[pyo3(signature = (*, model_rows, seeds, sources))] -fn declared_relation_keys<'py>( - py: Python<'py>, - model_rows: Vec, - seeds: &Bound<'py, PyAny>, - sources: &Bound<'py, PyAny>, -) -> PyResult> { - keys_dict(py, relation_keys(py, model_rows, (seeds, sources), &[])?) -} - -/// Declared keys of a compiled project's models, seeds and sources. -pub(crate) fn project_relation_keys(project: &Bound<'_, PyAny>) -> PyResult { - let py: Python<'_> = project.py(); - let mut rows: Vec = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { - rows.push(model_key_row(&model?)?); - } - let compiled_seeds: Vec> = native_field(project, "seeds")? - .try_iter()? - .collect::>()?; - let one_row_seeds: Vec = one_row_seed_names(&compiled_seeds)?; - let mut seeds: Vec> = Vec::new(); - for seed in &compiled_seeds { - seeds.push(native_field(seed, "schema_entry")?); - } - let mut sources: Vec> = Vec::new(); - for source in native_field(project, "sources")?.try_iter()? { - sources.push(source_key_input(&source?)?); - } - relation_keys( - py, - rows, - ( - PyTuple::new(py, seeds)?.as_any(), - PyTuple::new(py, sources)?.as_any(), - ), - &one_row_seeds, - ) -} - -/// Declared keys of a compiled project's models, seeds and sources. -#[pyfunction] -fn compiled_relation_keys<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - keys_dict(project.py(), project_relation_keys(project)?) -} - -fn declared_relation(row: RelationKeyRow) -> DeclaredRelation { - let RelationKeyRow { - kind, - name, - materialized, - mode, - strategy, - has_cursor, - unique_key, - columns, - audits, - } = row; - DeclaredRelation { - kind, - name, - materialized, - incremental_mode: mode, - incremental_strategy: strategy, - has_cursor, - unique_key, - columns: columns - .into_iter() - .map(|(name, audits, not_nullable)| DeclaredKeyColumn { - name, - audits, - not_nullable, - }) - .collect(), - audits, - at_most_one_row: false, - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(declared_relation_keys, module)?)?; - module.add_function(wrap_pyfunction!(compiled_relation_keys, module)?) -} - -/// The declared key facts of a model input or compiled model. -pub(crate) fn model_key_row(model: &Bound<'_, PyAny>) -> PyResult { - let py: Python<'_> = model.py(); - if let Ok(input) = model.downcast::() { - return input.get().read_with_owner(py, |owner, record| { - model_relation_key_row(owner, record, py) - }); - } - if let Ok(compiled) = model.downcast::() { - return compiled.get().relation_key_row(py); - } - native_field(model, "relation_key_row")?.extract() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/resource_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/resource_reads.rs deleted file mode 100644 index 95ebe5b0d6..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/resource_reads.rs +++ /dev/null @@ -1,152 +0,0 @@ -//! A compiled resource's attributes read alike for models, functions, seeds and sources. - -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; - -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - NativeCompiledModel, model_graph_row, -}; -use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; -use crate::bindings::_helpers::compiled_views::seed_views::NativeCompiledSeed; -use crate::bindings::_helpers::compiled_views::source_views::NativeCompiledSource; -use crate::bindings::_helpers::graph::keys::native_key; -use crate::bindings::_helpers::graph::project_graph::ResourceRow; -use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; -use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -/// A resource's graph facts without constructing its public keys or dependency tuple. -pub(crate) fn graph_row(resource: &Bound<'_, PyAny>) -> PyResult { - let py = resource.py(); - if let Ok(view) = resource.cast::() { - return model_graph_row(view.get(), py); - } - if let Ok(view) = resource.cast::() { - return view.get().graph_row(py); - } - if let Ok(view) = resource.cast::() { - return view.get().graph_row(py); - } - if let Ok(view) = resource.cast::() { - return view.get().graph_row(py); - } - count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; - Ok(ResourceRow { - key: native_key(&native_field(resource, "key")?)?, - deps: native_field(resource, "deps")? - .try_iter()? - .map(|key| native_key(&key?)) - .collect::>()?, - tags: Vec::new(), - folder: None, - }) -} - -/// A resource's `destination` from whichever native view it is; any other object is read by attribute, counted. -pub(crate) fn destination<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = resource.py(); - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledModel = view.get(); - return view - .destination(py) - .and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledFunction = view.get(); - return view - .destination(py) - .and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledSeed = view.get(); - return view - .destination(py) - .and_then(|value| value.into_bound_py_any(py)); - } - count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; - native_field(resource, "destination") -} - -/// A resource's `key` from whichever native view it is; any other object is read by attribute, counted. -pub(crate) fn key<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = resource.py(); - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledModel = view.get(); - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledFunction = view.get(); - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledSeed = view.get(); - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledSource = view.get(); - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; - native_field(resource, "key") -} - -/// A resource's `name` from whichever native view it is; any other object is read by attribute, counted. -pub(crate) fn name<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = resource.py(); - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledModel = view.get(); - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledFunction = view.get(); - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledSeed = view.get(); - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledSource = view.get(); - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; - native_field(resource, "name") -} - -/// A resource's `references` from whichever native view it is; any other object is read by attribute, counted. -pub(crate) fn references<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = resource.py(); - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledModel = view.get(); - return view - .references(py) - .and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledFunction = view.get(); - return view - .references(py) - .and_then(|value| value.into_bound_py_any(py)); - } - count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; - native_field(resource, "references") -} - -/// A resource's `schema_entry` from whichever native view it is; any other object is read by attribute, counted. -pub(crate) fn schema_entry<'py>(resource: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = resource.py(); - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledModel = view.get(); - return view - .schema_entry(py) - .and_then(|value| value.into_bound_py_any(py)); - } - if let Ok(view) = resource.cast::() { - let view: &NativeCompiledSeed = view.get(); - return view - .schema_entry(py) - .and_then(|value| value.into_bound_py_any(py)); - } - count_stage(py, COMPILED_VIEW_FALLBACKS, 1)?; - native_field(resource, "schema_entry") -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_reads.rs deleted file mode 100644 index 1912c33f0f..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_reads.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! A compiled seed's attributes the planner reads, from its native view. - -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; - -use crate::bindings::_helpers::compiled_views::seed_views::NativeCompiledSeed; -use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; -use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -/// The native view behind a seed object; any other object is a Python-built record, counted. -#[inline] -fn native_seed<'a>(seed: &'a Bound<'_, PyAny>) -> PyResult> { - if let Ok(view) = seed.cast::() { - return Ok(Some(view.get())); - } - count_stage(seed.py(), COMPILED_VIEW_FALLBACKS, 1)?; - Ok(None) -} - -/// The compiled seed's `destination`, as its Python attribute reads it. -pub(crate) fn destination<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = seed.py(); - if let Some(view) = native_seed(seed)? { - return view - .destination(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(seed, "destination") -} - -/// The compiled seed's `external`, as its Python attribute reads it. -pub(crate) fn external<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = seed.py(); - if let Some(view) = native_seed(seed)? { - return view.is_external().into_bound_py_any(py); - } - native_field(seed, "external") -} - -/// The compiled seed's `key`, as its Python attribute reads it. -pub(crate) fn key<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = seed.py(); - if let Some(view) = native_seed(seed)? { - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(seed, "key") -} - -/// The compiled seed's `name`, as its Python attribute reads it. -pub(crate) fn name<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = seed.py(); - if let Some(view) = native_seed(seed)? { - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(seed, "name") -} - -/// The compiled seed's `schema_entry`, as its Python attribute reads it. -pub(crate) fn schema_entry<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = seed.py(); - if let Some(view) = native_seed(seed)? { - return view - .schema_entry(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(seed, "schema_entry") -} - -/// The compiled seed's `seed_file`, as its Python attribute reads it. -pub(crate) fn seed_file<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = seed.py(); - if let Some(view) = native_seed(seed)? { - return view - .seed_file(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(seed, "seed_file") -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs deleted file mode 100644 index 1c898d7800..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs +++ /dev/null @@ -1,350 +0,0 @@ -//! Seeds paired with their declarations, held by the compiled project, and their views. - -use pyo3::exceptions::{PyIndexError, PyRuntimeError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::PyTuple; -use pyo3::{IntoPyObjectExt, pyclass, pymethods}; -use sqlbuild_attachments::seeds::main::pair_seed_declarations::pair_seed_declarations; -use sqlbuild_python_base::positional_record; - -use sqlbuild_project::models as project_models; - -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - defaulted_destination, object_key, relation_location, -}; -use crate::bindings::_helpers::compiled_views::model_input_view::{InputViewClasses, view_classes}; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::graph::project_graph::ResourceRow; -use sqlbuild_project::models::DestinationFact; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -const SEED_RESOURCE: &str = "seed"; -const TAGS_ATTRIBUTE: &str = "tags"; -const NAME_ATTRIBUTE: &str = "name"; -const FILE_PATH_ATTRIBUTE: &str = "file_path"; -const STEM_ATTRIBUTE: &str = "stem"; -const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; - -positional_record! { - extract; - /// `(database, schema, logical database, logical schema)` of a seed's destination. - pub(crate) struct SeedNamespace { - database: Option, - schema: Option, - logical_database: Option, - logical_schema: Option, - } -} - -/// A seed's name, CSV file, declaration and declaring schema file. -pub(crate) struct SeedParts<'a> { - pub(crate) name: &'a str, - pub(crate) seed_file: &'a Py, - pub(crate) schema_entry: &'a Py, - pub(crate) schema_file: &'a Py, -} - -/// One seed CSV paired with its declaration; the discovered objects are discovery's. -pub(crate) type SeedRecord = project_models::SeedRecord; - -/// A seed's discovered objects and the key its views build. -pub(crate) struct SeedHandles { - /// The discovered CSV file. - seed_file: Py, - /// The discovered declaration, whose spec carries user-authored column metadata. - schema_entry: Py, - /// The schema file declaring the seed. - schema_file: Py, - key: PyOnceLock>, -} - -/// What a seed record reads and keeps through its Python values. -pub(crate) trait SeedViews { - /// The seed's name, CSV file, declaration and declaring schema file. - fn parts(&self) -> SeedParts<'_>; - - /// Keep the destination native project assembly resolved for the seed. - fn set_namespace(&mut self, namespace: SeedNamespace); -} - -/// Pair each declaration with the CSV file of its name, raising for the first without one. -pub(crate) fn paired_seeds( - declarations: Vec<(Py, Py)>, - seed_files: &[Py], - compile_error: &Bound<'_, PyAny>, -) -> PyResult> { - let py: Python<'_> = compile_error.py(); - let mut names: Vec = Vec::with_capacity(declarations.len()); - for (entry, _) in &declarations { - names.push(native_field(entry.bind(py), NAME_ATTRIBUTE)?.extract()?); - } - let mut stems: Vec = Vec::with_capacity(seed_files.len()); - for seed_file in seed_files { - stems.push( - native_field( - &native_field(seed_file.bind(py), FILE_PATH_ATTRIBUTE)?, - STEM_ATTRIBUTE, - )? - .extract()?, - ); - } - let pairs: Vec = match pair_seed_declarations(&names, &stems) { - Ok(pairs) => pairs, - Err(missing) => { - let schema_file: &Py = &declarations[missing].1; - let relative_path: String = - native_field(schema_file.bind(py), RELATIVE_PATH_ATTRIBUTE)? - .str()? - .extract()?; - let message: String = format!( - "Seed declaration '{}' in {relative_path} has no matching CSV file under seeds/", - names[missing] - ); - return Err(PyErr::from_value(compile_error.call1((message,))?)); - } - }; - let mut seeds: Vec = Vec::with_capacity(pairs.len()); - for (((schema_entry, schema_file), name), file_index) in - declarations.into_iter().zip(names).zip(pairs) - { - seeds.push(SeedRecord { - name, - destination: None, - host: SeedHandles { - seed_file: seed_files[file_index].clone_ref(py), - schema_entry, - schema_file, - key: PyOnceLock::new(), - }, - }); - } - Ok(seeds) -} - -impl SeedViews for SeedRecord { - fn parts(&self) -> SeedParts<'_> { - SeedParts { - name: &self.name, - seed_file: &self.host.seed_file, - schema_entry: &self.host.schema_entry, - schema_file: &self.host.schema_file, - } - } - - fn set_namespace(&mut self, namespace: SeedNamespace) { - let SeedNamespace { - database, - schema, - logical_database, - logical_schema, - } = namespace; - self.destination = Some(DestinationFact { - database, - schema, - name: self.name.clone(), - qualified_name: None, - logical_schema, - logical_database, - }); - } -} - -fn read_seed<'py, T>( - owner: &Py, - index: usize, - py: Python<'py>, - read: impl FnOnce(&SeedRecord, Python<'py>) -> PyResult, -) -> PyResult { - let owner = owner.bind(py).borrow(); - let seed: &SeedRecord = owner - .seeds() - .get(index) - .ok_or_else(|| PyIndexError::new_err(index))?; - read(seed, py) -} - -/// One attached seed, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeSeedInput { - owner: Py, - index: usize, -} - -impl NativeSeedInput { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } -} - -#[pymethods] -impl NativeSeedInput { - #[getter] - pub(crate) fn seed_file(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.seed_file.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn schema_entry(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.schema_entry.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn schema_file(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.schema_file.clone_ref(py)) - }) - } -} - -/// One assembled seed, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeCompiledSeed { - owner: Py, - index: usize, - /// The destination with target defaults applied, replacing the assembled one. - defaulted: Option, - destination: PyOnceLock>, -} - -impl NativeCompiledSeed { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { - owner, - index, - defaulted: None, - destination: PyOnceLock::new(), - } - } - - /// The seed's project graph row: its key and its declared tags. - pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { - read_seed(&self.owner, self.index, py, |seed, py| { - let mut tags: Vec = Vec::new(); - for tag in native_field(seed.host.schema_entry.bind(py), TAGS_ATTRIBUTE)?.try_iter()? { - tags.push(tag?.str()?.extract()?); - } - Ok(ResourceRow { - key: (SEED_RESOURCE.to_owned(), seed.name.clone()), - deps: Vec::new(), - tags, - folder: None, - }) - }) - } - - pub(crate) fn current_destination(&self, py: Python<'_>) -> PyResult { - if let Some(defaulted) = &self.defaulted { - return Ok(defaulted.clone()); - } - read_seed(&self.owner, self.index, py, |seed, _| { - seed.destination - .clone() - .ok_or_else(|| PyRuntimeError::new_err("the seed was not assembled")) - }) - } -} - -#[pymethods] -impl NativeCompiledSeed { - /// The seed's planner-facing `CompiledObjectKey`. - #[getter] - pub(crate) fn key(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - read_seed(&self.owner, self.index, py, |seed, py| { - seed.host - .key - .get_or_try_init(py, || { - object_key(&classes.compiled, SEED_RESOURCE, &seed.name) - }) - .map(|key| key.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn deps<'py>(&self, py: Python<'py>) -> Bound<'py, PyTuple> { - PyTuple::empty(py) - } - - #[getter] - pub(crate) fn name(&self, py: Python<'_>) -> PyResult { - read_seed(&self.owner, self.index, py, |seed, _| Ok(seed.name.clone())) - } - - #[getter] - pub(crate) fn seed_file(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.seed_file.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn schema_entry(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.schema_entry.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn schema_file(&self, py: Python<'_>) -> PyResult> { - read_seed(&self.owner, self.index, py, |seed, py| { - Ok(seed.host.schema_file.clone_ref(py)) - }) - } - - /// The seed's planner-facing `CompiledRelationLocation`. - #[getter] - pub(crate) fn destination(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - self.destination - .get_or_try_init(py, || { - relation_location(&classes.compiled, &self.current_destination(py)?) - }) - .map(|destination| destination.clone_ref(py)) - } - - /// This seed with the adapter's default schema and database filled into its destination. - pub(crate) fn with_target_defaults( - &self, - py: Python<'_>, - defaults: (Option, Option), - render: Bound<'_, PyAny>, - ) -> PyResult { - let defaulted: DestinationFact = - defaulted_destination(&self.current_destination(py)?, defaults, &render)?; - Ok(Self { - owner: self.owner.clone_ref(py), - index: self.index, - defaulted: Some(defaulted), - destination: PyOnceLock::new(), - }) - } - - #[getter(external)] - pub(crate) fn is_external(&self) -> bool { - false - } -} - -impl NativeCompiledSeed { - /// The `CompiledSeed` field values, in field order, without building the record. - pub(crate) fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { - Ok(vec![ - self.key(py)?.into_bound(py), - self.deps(py).into_any(), - self.name(py)?.into_bound_py_any(py)?, - self.seed_file(py)?.into_bound(py), - self.schema_entry(py)?.into_bound(py), - self.schema_file(py)?.into_bound(py), - self.destination(py)?.into_bound(py), - pyo3::types::PyBool::new(py, self.is_external()) - .to_owned() - .into_any(), - ]) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_reads.rs deleted file mode 100644 index 46dc34fa56..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_reads.rs +++ /dev/null @@ -1,59 +0,0 @@ -//! A compiled source's attributes the planner reads, from its native view. - -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; - -use crate::bindings::_helpers::compiled_views::source_views::NativeCompiledSource; -use sqlbuild_python_base::compile_driver::constants::COMPILED_VIEW_FALLBACKS; -use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -/// The native view behind a source object; any other object is a Python-built record, counted. -#[inline] -fn native_source<'a>(source: &'a Bound<'_, PyAny>) -> PyResult> { - if let Ok(view) = source.cast::() { - return Ok(Some(view.get())); - } - count_stage(source.py(), COMPILED_VIEW_FALLBACKS, 1)?; - Ok(None) -} - -/// The compiled source's `key`, as its Python attribute reads it. -pub(crate) fn key<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = source.py(); - if let Some(view) = native_source(source)? { - return view.key(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(source, "key") -} - -/// The compiled source's `name`, as its Python attribute reads it. -pub(crate) fn name<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = source.py(); - if let Some(view) = native_source(source)? { - return view.name(py).and_then(|value| value.into_bound_py_any(py)); - } - native_field(source, "name") -} - -/// The compiled source's `source_entry`, as its Python attribute reads it. -pub(crate) fn source_entry<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = source.py(); - if let Some(view) = native_source(source)? { - return view - .source_entry(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(source, "source_entry") -} - -/// The compiled source's `source_file`, as its Python attribute reads it. -pub(crate) fn source_file<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = source.py(); - if let Some(view) = native_source(source)? { - return view - .source_file(py) - .and_then(|value| value.into_bound_py_any(py)); - } - native_field(source, "source_file") -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs deleted file mode 100644 index ef3d784416..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs +++ /dev/null @@ -1,333 +0,0 @@ -//! Sources attached by the native loop, held by the compiled project, and their views. - -use pyo3::exceptions::PyIndexError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, pyclass, pymethods}; -use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; -use sqlbuild_python_base::positional_record; - -use sqlbuild_project::models as project_models; - -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; -use crate::bindings::_helpers::compiled_views::compiled_model_view::object_key; -use crate::bindings::_helpers::compiled_views::model_input_view::{InputViewClasses, view_classes}; -use crate::bindings::_helpers::compiled_views::project_host::PyProjectHost; -use crate::bindings::_helpers::compiled_views::relation_keys::audit_key_rows; -use crate::bindings::_helpers::graph::project_graph::ResourceRow; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; - -const SOURCE_RESOURCE: &str = "source"; - -positional_record! { - into; - /// `(name, has a loader, database, schema)` of one source. - pub(crate) struct SourceAssemblyRow { - name: String, - managed: bool, - database: Option, - schema: Option, - } -} - -/// What the source loop attached for one source: its expanded metadata and usages. -pub(crate) type AttachedSource = project_models::AttachedSource; - -/// One attached source held by the compiled project. -pub(crate) type SourceRecord = project_models::SourceRecord; - -/// An attached source's user values. -pub(crate) struct SourceHandles { - /// The discovered source file. - pub(crate) source_file: Py, - /// The discovered entry; its spec carries user-authored metadata. - pub(crate) entry: Py, - /// Template-expanded user metadata. - pub(crate) meta: Py, - /// Template-expanded column specs, which carry user metadata and audit arguments. - pub(crate) columns: Py, - /// Template-expanded table audit specs, which carry user audit arguments. - pub(crate) audits: Py, - /// Usage records, the identities the scope index is built from. - pub(crate) usages: Py, -} - -/// What a source declares about its keys: its column facts and `(audit, column argument)` rows. -pub(crate) struct SourceKeyFacts { - pub(crate) columns: Vec, - pub(crate) audits: Vec<(String, Option)>, -} - -/// A source record's key facts and entries, each built once from its Python values. -pub(crate) struct SourceProjections { - key_facts: PyOnceLock, - attached_entry: PyOnceLock>, - assembled_entry: PyOnceLock>, - pub(crate) key: PyOnceLock>, -} - -/// The source record of an attached source, before assembly resolves its namespace. -pub(crate) fn source_record(attached: AttachedSource) -> SourceRecord { - SourceRecord { - attached, - namespace: None, - host: SourceProjections { - key_facts: PyOnceLock::new(), - attached_entry: PyOnceLock::new(), - assembled_entry: PyOnceLock::new(), - key: PyOnceLock::new(), - }, - } -} - -/// What a source record projects from its Python values, each built once. -pub(crate) trait SourceViews { - /// The source's declared column facts and audit key rows, read once from its expanded specs. - fn key_facts(&self, py: Python<'_>) -> PyResult<&SourceKeyFacts>; - - /// The attached `SourceEntry`: the discovered entry with its expanded metadata. - fn attached_entry(&self, py: Python<'_>) -> PyResult>; - - /// The assembled `SourceEntry`: the attached entry in its resolved namespace. - fn assembled_entry(&self, py: Python<'_>) -> PyResult>; -} - -impl SourceViews for SourceRecord { - fn key_facts(&self, py: Python<'_>) -> PyResult<&SourceKeyFacts> { - self.host.key_facts.get_or_try_init(py, || { - Ok(SourceKeyFacts { - columns: python_column_facts(self.attached.host.columns.bind(py).as_any(), None)?, - audits: audit_key_rows(self.attached.host.audits.bind(py).as_any())?, - }) - }) - } - - fn attached_entry(&self, py: Python<'_>) -> PyResult> { - self.host - .attached_entry - .get_or_try_init(py, || { - let source: &AttachedSource = &self.attached; - let changes: Bound<'_, PyDict> = PyDict::new(py); - changes.set_item("contract", &source.contract)?; - changes.set_item("database", &source.database)?; - changes.set_item("schema", &source.schema)?; - changes.set_item("table", &source.table)?; - changes.set_item("expression", &source.expression)?; - changes.set_item("description", &source.description)?; - changes.set_item("meta", source.host.meta.bind(py))?; - changes.set_item("columns", source.host.columns.bind(py))?; - changes.set_item("audits", source.host.audits.bind(py))?; - replaced(py, source.host.entry.bind(py), &changes) - }) - .map(|entry| entry.clone_ref(py)) - } - - fn assembled_entry(&self, py: Python<'_>) -> PyResult> { - let attached: Py = self.attached_entry(py)?; - let Some(Some((database, schema))) = &self.namespace else { - return Ok(attached); - }; - self.host - .assembled_entry - .get_or_try_init(py, || { - let changes: Bound<'_, PyDict> = PyDict::new(py); - changes.set_item("database", database)?; - changes.set_item("schema", schema)?; - replaced(py, attached.bind(py), &changes) - }) - .map(|entry| entry.clone_ref(py)) - } -} - -/// `dataclasses.replace(value, **changes)`. -pub(crate) fn replaced( - _py: Python<'_>, - value: &Bound<'_, PyAny>, - changes: &Bound<'_, PyDict>, -) -> PyResult> { - Ok(replaced_keywords(value, changes)?.unbind()) -} - -/// Read the source record behind a native source view, or `None` for any other object. -pub(crate) fn read_source_view<'py, T>( - view: &Bound<'py, PyAny>, - read: impl FnOnce(&SourceRecord, Python<'py>) -> PyResult, -) -> Option> { - let py: Python<'py> = view.py(); - if let Ok(input) = view.cast::() { - let input: &NativeSourceInput = input.get(); - return Some(read_source(&input.owner, input.index, py, read)); - } - let Ok(compiled) = view.cast::() else { - return None; - }; - let compiled: &NativeCompiledSource = compiled.get(); - Some(read_source(&compiled.owner, compiled.index, py, read)) -} - -fn read_source<'py, T>( - owner: &Py, - index: usize, - py: Python<'py>, - read: impl FnOnce(&SourceRecord, Python<'py>) -> PyResult, -) -> PyResult { - let owner = owner.bind(py).borrow(); - let source: &SourceRecord = owner - .sources() - .get(index) - .ok_or_else(|| PyIndexError::new_err(index))?; - read(source, py) -} - -/// One attached source, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeSourceInput { - owner: Py, - index: usize, -} - -impl NativeSourceInput { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } -} - -#[pymethods] -impl NativeSourceInput { - #[getter] - pub(crate) fn source_entry(&self, py: Python<'_>) -> PyResult> { - read_source(&self.owner, self.index, py, |source, py| { - source.attached_entry(py) - }) - } - - #[getter] - pub(crate) fn source_file(&self, py: Python<'_>) -> PyResult> { - read_source(&self.owner, self.index, py, |source, py| { - Ok(source.attached.host.source_file.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn declaration_usages(&self, py: Python<'_>) -> PyResult> { - read_source(&self.owner, self.index, py, |source, py| { - Ok(source.attached.host.usages.clone_ref(py)) - }) - } - - /// `(name, has a loader, database, schema)`, what project assembly resolves a namespace from. - #[getter] - pub(crate) fn assembly_row(&self, py: Python<'_>) -> PyResult { - read_source(&self.owner, self.index, py, |source, py| { - let attached: &AttachedSource = &source.attached; - Ok(SourceAssemblyRow { - name: attached.name.clone(), - managed: !native_field(attached.host.entry.bind(py), "loader")?.is_none(), - database: attached.database.clone(), - schema: attached.schema.clone(), - }) - }) - } -} - -/// One assembled source, read from the compiled project that owns it. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeCompiledSource { - owner: Py, - index: usize, -} - -impl NativeCompiledSource { - pub(crate) fn new(owner: Py, index: usize) -> Self { - Self { owner, index } - } - - /// The source's project graph row: its key, with no dependencies, tags or folder. - pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { - read_source(&self.owner, self.index, py, |source, _| { - Ok(ResourceRow { - key: (SOURCE_RESOURCE.to_owned(), source.attached.name.clone()), - deps: Vec::new(), - tags: Vec::new(), - folder: None, - }) - }) - } -} - -#[pymethods] -impl NativeCompiledSource { - /// The source's planner-facing `CompiledObjectKey`. - #[getter] - pub(crate) fn key(&self, py: Python<'_>) -> PyResult> { - let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - read_source(&self.owner, self.index, py, |source, py| { - source - .host - .key - .get_or_try_init(py, || { - object_key(&classes.compiled, SOURCE_RESOURCE, &source.attached.name) - }) - .map(|key| key.clone_ref(py)) - }) - } - - #[getter] - pub(crate) fn deps<'py>(&self, py: Python<'py>) -> Bound<'py, PyTuple> { - PyTuple::empty(py) - } - - #[getter] - pub(crate) fn name(&self, py: Python<'_>) -> PyResult { - read_source(&self.owner, self.index, py, |source, _| { - Ok(source.attached.name.clone()) - }) - } - - #[getter] - pub(crate) fn source_entry(&self, py: Python<'_>) -> PyResult> { - read_source(&self.owner, self.index, py, |source, py| { - source.assembled_entry(py) - }) - } - - #[getter] - pub(crate) fn source_file(&self, py: Python<'_>) -> PyResult> { - read_source(&self.owner, self.index, py, |source, py| { - Ok(source.attached.host.source_file.clone_ref(py)) - }) - } -} - -impl NativeCompiledSource { - /// The `CompiledSource` field values, in field order, without building the record. - pub(crate) fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { - Ok(vec![ - self.key(py)?.into_bound(py), - self.deps(py).into_any(), - self.name(py)?.into_bound_py_any(py)?, - self.source_entry(py)?.into_bound(py), - self.source_file(py)?.into_bound(py), - ]) - } -} - -/// Keep `attached` in `owner`; return each source's view. -pub(crate) fn retain_sources( - owner: &Bound<'_, NativeCompiledProject>, - attached: Vec, -) -> Vec { - let mut project = owner.borrow_mut(); - let first: usize = project.sources().len(); - let count: usize = attached.len(); - for source in attached { - project.retain_source(source_record(source)); - } - drop(project); - (first..first + count) - .map(|index| NativeSourceInput::new(owner.clone().unbind(), index)) - .collect() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs index 88801d0be6..ebd65842a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/facts.rs @@ -7,19 +7,25 @@ use pyo3::types::PyDict; use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; use sqlbuild_state::consumption::models::{ForcedRun, InputFact, NodeFacts, NodeKey, RelationKey}; -use crate::bindings::_helpers::compiled_views::{ - function_reads, model_reads, project_reads, seed_reads, -}; use crate::bindings::_helpers::consumption::relation_reads::{ Locations, relation, sql_reads_undeclared, }; -use crate::bindings::_helpers::planner_identity::hashing::{baselined_model_query_hash, graph_key}; use crate::bindings::_helpers::planner_identity::identities::hook_version_hashes; use crate::bindings::_helpers::planner_identity::metadata::{ function_local_hashes, model_version_identity_metadata_json, seed_identity, }; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_project::compiled_views::main::compiled_models::dep_rows::dep_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; +use sqlbuild_python_project::planner_identity::main::baselined_model_query_hash::baselined_model_query_hash; +use sqlbuild_python_project::planner_identity::main::graph_key::graph_key; +use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; +use sqlbuild_python_project::resource_views::main::function_reads::deps::deps; +use sqlbuild_python_project::resource_views::main::function_reads::language::language; pub(crate) const MODEL: &str = "model"; pub(crate) const SEED: &str = "seed"; @@ -48,8 +54,8 @@ enum Compiled<'py> { impl<'py> Compiled<'py> { fn destination(&self) -> PyResult> { match self { - Self::Model(model) => model_reads::destination(model), - Self::Seed(seed) => seed_reads::destination(seed), + Self::Model(model) => sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(model), + Self::Seed(seed) => sqlbuild_python_project::resource_views::main::seed_reads::destination::destination(seed), } } } @@ -221,41 +227,47 @@ pub(crate) fn build_facts( ) -> PyResult { let py: Python<'_> = plan.py(); let mut compiled: HashMap> = HashMap::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model: Bound<'_, PyAny> = model?; compiled.insert( - node_key(MODEL, &model_reads::model_name(&model)?), + node_key(MODEL, &model_name(&model)?), Compiled::Model(model), ); } - for seed in project_reads::seeds(project)?.try_iter()? { + for seed in seeds(project)?.try_iter()? { let seed: Bound<'_, PyAny> = seed?; compiled.insert( - node_key(SEED, &seed_reads::name(&seed)?.extract::()?), + node_key( + SEED, + &sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)? + .extract::()?, + ), Compiled::Seed(seed), ); } let mut function_deps: HashMap<(String, String), Vec<(String, String)>> = HashMap::new(); let mut function_bodies: HashMap<(String, String), Option> = HashMap::new(); - let functions: Bound<'_, PyAny> = project_reads::functions(project)?; + let functions: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)?; for function in functions.try_iter()? { let function: Bound<'_, PyAny> = function?; - let key = graph_key(&function_reads::key(&function)?)?; + let key = graph_key( + &sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?, + )?; function_deps.insert( (key.kind.clone(), key.name.clone()), - key_rows(&function_reads::deps(&function)?)?, + key_rows(&deps(&function)?)?, ); - let body: Option = - if enum_text(&function_reads::language(&function)?)? == SQL_LANGUAGE { - let body: String = function_reads::body_sql(&function)?.extract()?; - Some(if key.kind == UDF { - format!("SELECT ({body})") - } else { - body - }) + let body: Option = if enum_text(&language(&function)?)? == SQL_LANGUAGE { + let body: String = body_sql(&function)?.extract()?; + Some(if key.kind == UDF { + format!("SELECT ({body})") } else { - None - }; + body + }) + } else { + None + }; function_bodies.insert((key.kind, key.name), body); } let mut selected: BTreeMap> = BTreeMap::new(); @@ -312,15 +324,12 @@ pub(crate) fn build_facts( }; let (deps, version_hash) = match node { Compiled::Model(model) => ( - model_reads::dep_rows(model)?, + dep_rows(model)?, model_local_identity_hash( &baselined_model_query_hash( py, &key.name, - ( - &model_reads::query_sql(model)?.extract::()?, - dialect.as_deref(), - ), + (&query_sql(model)?.extract::()?, dialect.as_deref()), &mapping_get(&recorded_models, &key.name)?, )?, &model_version_identity_metadata_json( @@ -347,7 +356,7 @@ pub(crate) fn build_facts( (node, &locations, &mut function_checks) && (sql_reads_undeclared( py, - model_reads::query_sql(model)?.extract()?, + query_sql(model)?.extract()?, (&node_inputs, relation.as_ref()), (locations, dialect.as_deref()), )? || function_checks.undeclared(py, &deps)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs index 97b34ea561..299520c88f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/policy.rs @@ -9,10 +9,10 @@ use sqlbuild_state::consumption::models::{ ChangeKind, ChangedPolicy, NodeKey, Reach, TriggeredJob, }; -use crate::bindings::_helpers::compiled_project::job_catalog::triggered_jobs; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; +use sqlbuild_python_project::compiled_project::main::triggered_jobs::triggered_jobs; +use sqlbuild_python_project::graph::models::NativeProjectGraph; const DEFINITION: &str = "definition"; const SOURCE_DATA: &str = "source_data"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs index 54655ad6ed..ecf1b8c59c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/recorder.rs @@ -24,9 +24,9 @@ use crate::bindings::_helpers::consumption::facts::{ BuildFacts, MODEL, SEED, build_facts, enum_text, }; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::native_host::models::ConsumptionConnection; -use sqlbuild_python_front::native_host::models::Location; -use sqlbuild_python_front::native_host::models::NodeCompleteHook; +use sqlbuild_python_front::native_host::models::{ + ConsumptionConnection, Location, NodeCompleteHook, +}; /// Succeeded rows are flushed once this many executions are pending. const FLUSH_EXECUTIONS: usize = 100; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs index 2b1e918e9e..d5f910f5d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/consumption/relation_reads.rs @@ -6,12 +6,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_analysis::auditing::main::replace_reference_calls::replace_reference_calls; use sqlbuild_state::consumption::models::{InputFact, NodeKey, RelationKey}; -use crate::bindings::_helpers::analysis::relation_names::{ - RelationRow, extract_sql_relation_names, -}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_front::native_host::main::optional_text::optional_text; +use sqlbuild_python_project::analysis::main::extract_sql_relation_names::extract_sql_relation_names; +use sqlbuild_python_project::analysis::types::RelationRow; /// Function calls, `__udf("name")(...)`, parsed as plain calls; their reads are declared inputs. const FUNCTION_CALLS: [&str; 2] = ["__udf", "__table_fn"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index 8ac95d3422..557235e6ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -24,8 +24,6 @@ use crate::bindings::_helpers::compile_driver::discovery::discover_project_input use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::contract_command::write::{ContractWrite, write_contracts}; -use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -38,6 +36,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::refactor::main::compare_contracts::compare_contracts; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; +use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; const GENERATE_ACTION: &str = "generate"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs index 375db96d18..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs @@ -1,4 +1 @@ -pub(crate) mod contract_diagnostics; -pub(crate) mod exception_types; -pub(crate) mod model_contracts; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs index a1f676c557..36bea04a1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs @@ -2,18 +2,19 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::contracts::{contract_diagnostics, exception_types}; use sqlbuild_python_base::contracts::main::register_resource_identities::register_resource_identities; use sqlbuild_python_base::contracts::main::register_state_literals::register_state_literals; use sqlbuild_python_front::authored_values::main::register_sql_values::register_sql_values; use sqlbuild_python_front::contracts::main::register_effective_config::register_effective_config; use sqlbuild_python_front::contracts::main::register_spec_resolution::register_spec_resolution; use sqlbuild_python_front::discovery::main::project::register_project_config::register_project_config; +use sqlbuild_python_project::contracts::main::register_contract_diagnostics::register_contract_diagnostics; +use sqlbuild_python_project::contracts::main::register_exception_types::register_exception_types; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - contract_diagnostics::register(module)?; + register_contract_diagnostics(module)?; register_effective_config(module)?; - exception_types::register(module)?; + register_exception_types(module)?; register_project_config(module)?; register_resource_identities(module)?; register_spec_resolution(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs index 0d6efc02e9..cee82aa1ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs @@ -9,10 +9,10 @@ use sqlbuild_analysis::dag_artifact::models::{DagProjectInput, DagPythonGraphInp use sqlbuild_core::json::errors::JsonEmitError; use crate::bindings::_helpers::dag_artifact::rows::{project_input, python_graph_input}; -use crate::bindings::_helpers::graph::project_graph::CompiledProjectGraph; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_project::graph::models::CompiledProjectGraph; fn python_error(error: DagArtifactError, py: pyo3::Python<'_>) -> PyErr { match error { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs index a9ed9eb49c..f0bc584f2c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs @@ -17,11 +17,11 @@ use sqlbuild_analysis::dag_artifact::models::{ use sqlbuild_analysis::planner_selection::main::unit_test_selector_rejection::unit_test_selector_rejection; use sqlbuild_host::entries::models::ThirdPartyHost; -use crate::bindings::_helpers::planner_selection::carrier::Carrier; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; +use sqlbuild_python_project::planner_selection::models::Carrier; const PATHLIB_MODULE: &str = "pathlib"; const JSON_MODULE: &str = "json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs index 36f04c5968..956e86a5b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs @@ -20,9 +20,6 @@ use sqlbuild_core::dbt_graph::models::{ use sqlbuild_core::dbt_graph::types::DbtSelectorResolver; use crate::bindings::_helpers::dbt_interop::manifest::resolve_sqlbuild_model_dbt_refs; -use crate::bindings::_helpers::planner_selection::model_selectors::{ - resolve_sqlbuild_model_selector_names, split_selector_expansion, -}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -30,6 +27,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_selection::main::resolve_sqlbuild_model_selector_names::resolve_sqlbuild_model_selector_names; +use sqlbuild_python_project::planner_selection::main::split_selector_expansion::split_selector_expansion; const MODEL_RESOURCE_TYPE: &str = "model"; const SELECTION_OPTIONS: (&[&str], &[&str]) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index f34d93ed9b..11cd978ce0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -31,7 +31,6 @@ use crate::bindings::_helpers::diff_report::support::{ call, caused, cli_error, error_text, is_exception, matches_class, matches_native, none, object, request_facts, user_error_parts, }; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::frontier::main::emit_error::emit_error; @@ -43,6 +42,7 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_front::discovery::main::project::discover_project_configuration::discover_project_configuration; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; const DEFAULT_DIFF_MAX_FULL_ROWS: i64 = 10_000_000; const ADAPTER_ERRORS: AdapterHost = AdapterHost::AdapterUserError; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 2f51322b96..185030d8e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -47,10 +47,11 @@ use crate::bindings::_helpers::compile_delta::driver::{ use crate::bindings::_helpers::frontier::stage_capture::CAPTURE_DIR_ENV_VAR; use crate::bindings::_helpers::result_output::event_writer::RESULT_PATH_ENV; use crate::bindings::_helpers::rules::constants::CUSTOM_HOST_RUNTIME_VERSION; -use crate::bindings::_helpers::semantic_shapes::shapes::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; -use sqlbuild_python_front::attachments::constants::MICROBATCH_END_SENTINEL; -use sqlbuild_python_front::attachments::constants::MICROBATCH_START_SENTINEL; +use sqlbuild_python_front::attachments::constants::{ + MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, +}; use sqlbuild_python_front::state_rows::constants::MIGRATION_WRITE_ATTEMPTS; +use sqlbuild_python_project::semantic_shapes::constants::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; /// Text constants exported under their Python names. pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs index 846cc60777..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs @@ -1,5 +1 @@ -pub(crate) mod latest_rows; -pub(crate) mod query_hashes; pub(crate) mod registration; -pub(crate) mod storage_io; -pub(crate) mod storage_sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/registration.rs index 717efad9a0..36c1fdc4c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/registration.rs @@ -2,10 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::fingerprints::{latest_rows, query_hashes, storage_io}; +use sqlbuild_python_project::fingerprints::main::register_latest_rows::register_latest_rows; +use sqlbuild_python_project::fingerprints::main::register_query_hashes::register_query_hashes; +use sqlbuild_python_project::fingerprints::main::register_storage_io::register_storage_io; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - query_hashes::register(module)?; - storage_io::register(module)?; - latest_rows::register(module) + register_query_hashes(module)?; + register_storage_io(module)?; + register_latest_rows(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index 539fb0eec0..5b4028aeb3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -15,8 +15,6 @@ use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_ use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::boundary::main::within::within; @@ -36,6 +34,8 @@ use sqlbuild_python_front::discovery::main::records::discover_schema_files::disc use sqlbuild_python_front::discovery::main::records::discover_source_files::discover_source_files; use sqlbuild_python_front::discovery::main::records::discover_test_files::discover_test_files; use sqlbuild_python_front::discovery::main::snapshots::directory_snapshot_scope::directory_snapshot_scope; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; +use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; const SELECTION_CODE: &str = "S007"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 8c6b61c763..91312c9289 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -17,14 +17,9 @@ use sqlbuild_orchestration::runs::models::NodeOutcome; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::compiled_project::snapshot_execution::VerifiedRun; use crate::bindings::_helpers::freshness::freshness_observe::{PreviousState, observe_freshness}; use crate::bindings::_helpers::freshness::observation::Observer; use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; -use crate::bindings::_helpers::planner_selection::selectors::{ - expand_project_upstream_keys, select_project_graph, -}; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; @@ -32,6 +27,10 @@ use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_project::planner_selection::main::expand_project_upstream_keys::expand_project_upstream_keys; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; +use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; +use sqlbuild_python_project::project_snapshot::types::VerifiedRun; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME as STATE_TABLE_NAME; use sqlbuild_python_base::native_host::main::host_entry::host_entry; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs index 169e044634..873f93551f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs @@ -11,7 +11,6 @@ use crate::bindings::_helpers::freshness::support::{ OBSERVATION_ERROR, exception_text, freshness_error, presumed_utc_datetime, presumed_utc_value, text, }; -use crate::bindings::_helpers::references::source_relations::render_source_read; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -29,6 +28,7 @@ use sqlbuild_python_front::state_rows::main::event_rows::in_utc::in_utc; use sqlbuild_python_front::state_rows::main::event_rows::normalize::normalize; use sqlbuild_python_front::state_rows::main::event_rows::offset_aware::offset_aware; use sqlbuild_python_front::state_rows::main::event_rows::version_hash::version_hash; +use sqlbuild_python_project::references::main::render_source_read::render_source_read; const ADAPTER_STRATEGY: &str = "adapter"; const COLUMN_STRATEGY: &str = "column"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs index aabf7ddd0d..14b5a97b82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs @@ -20,7 +20,6 @@ use crate::bindings::_helpers::freshness::observation::{ }; use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; use crate::bindings::_helpers::freshness::support::text; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; @@ -32,6 +31,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::main::is_warehouse_access_refused::is_warehouse_access_refused; use sqlbuild_python_front::state_rows::main::event_rows::age_policy::age_policy; use sqlbuild_python_front::state_rows::main::event_rows::equivalent::equivalent; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; const ADAPTER_STRATEGY: &str = "adapter"; const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavailable"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs deleted file mode 100644 index 53b334f5bf..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs +++ /dev/null @@ -1,161 +0,0 @@ -//! Closure, edge inversion and path nodes over any Python-keyed edge mapping. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{ - PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PySet, PySetMethods, PyTuple, -}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; -use sqlbuild_core::graph_walk::main::directed_path_nodes::directed_path_nodes; - -use crate::bindings::_helpers::graph::node_set::PyNodeSet; -use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; - -const BUILTINS_MODULE: &str = "builtins"; - -/// Keys within `max_depth` edges of any start, plus the starts when `include_starts`. -pub(crate) fn closure<'py>( - starts: Vec>, - edges: &Bound<'py, PyAny>, - (include_starts, max_depth): (bool, Option), -) -> PyResult> { - let py: Python<'py> = edges.py(); - let mut visited: PyNodeSet<'py> = PyNodeSet(PySet::empty(py)?); - if include_starts { - for start in &starts { - visited.0.add(start)?; - } - } - let none: Bound<'py, PyTuple> = PyTuple::empty(py); - breadth_first_reach( - starts, - max_depth.map(|maximum| u64::try_from(maximum).unwrap_or(0)), - &mut visited, - |current| neighbours(edges, current, &none), - )?; - Ok(visited.0) -} - -/// The keys `edges.get(current, ())` lists. -fn neighbours<'py>( - edges: &Bound<'py, PyAny>, - current: &Bound<'py, PyAny>, - none: &Bound<'py, PyTuple>, -) -> PyResult>> { - mapping_get_or(edges, current, none)?.try_iter()?.collect() -} - -/// Keys within `max_depth` edges of any start, plus the starts when requested. -#[pyfunction] -#[pyo3(signature = (*, starts, edges, include_starts, max_depth = None))] -pub(crate) fn transitive_closure_many<'py>( - starts: &Bound<'py, PyAny>, - edges: &Bound<'py, PyAny>, - include_starts: bool, - max_depth: Option, -) -> PyResult> { - let starts: Vec> = starts.try_iter()?.collect::>()?; - let visited: Bound<'py, PySet> = closure(starts, edges, (include_starts, max_depth))?; - PyFrozenSet::new(edges.py(), visited.iter()) -} - -/// All graph keys reachable from a starting key. -#[pyfunction] -#[pyo3(signature = (*, start, edges, max_depth = None))] -fn transitive_closure<'py>( - start: Bound<'py, PyAny>, - edges: &Bound<'py, PyAny>, - max_depth: Option, -) -> PyResult> { - let visited: Bound<'py, PySet> = closure(vec![start], edges, (false, max_depth))?; - PyFrozenSet::new(edges.py(), visited.iter()) -} - -/// Reversed edges: every source key first, then each new target in first-seen order. -pub(crate) fn inverted<'py>(edges: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = edges.py(); - let inverted: Bound<'py, PyDict> = PyDict::new(py); - for key in edges.try_iter()? { - inverted.set_item(key?, PyList::empty(py))?; - } - for item in mapping_items(edges)?.try_iter()? { - let (source, targets): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; - for target in targets.try_iter()? { - let target: Bound<'py, PyAny> = target?; - let sources: Bound<'py, PyList> = match inverted.get_item(&target)? { - Some(sources) => sources.downcast_into()?, - None => { - let sources: Bound<'py, PyList> = PyList::empty(py); - inverted.set_item(&target, &sources)?; - sources - } - }; - sources.append(&source)?; - } - } - Ok(inverted) -} - -/// Reversed directed graph edges, each source list sorted by `sort_key` when given. -#[pyfunction] -#[pyo3(signature = (*, edges, sort_key = None))] -fn invert_edges<'py>( - edges: &Bound<'py, PyAny>, - sort_key: Option<&Bound<'py, PyAny>>, -) -> PyResult> { - let py: Python<'py> = edges.py(); - let inverted: Bound<'py, PyDict> = inverted(edges)?; - let sorted: Bound<'py, PyAny> = py.import(BUILTINS_MODULE)?.getattr("sorted")?; - let result: Bound<'py, PyDict> = PyDict::new(py); - for (key, sources) in inverted.iter() { - let sources: Bound<'py, PyAny> = match sort_key { - None => sources, - Some(sort_key) => { - let keywords: Bound<'py, PyDict> = PyDict::new(py); - keywords.set_item("key", sort_key)?; - sorted.call((sources,), Some(&keywords))? - } - }; - result.set_item( - key, - PyTuple::new(py, sources.try_iter()?.collect::>>()?)?, - )?; - } - Ok(result) -} - -/// Keys on directed paths between two endpoints, or `None` when `end` is not downstream. -#[pyfunction] -#[pyo3(signature = (*, start, end, downstream))] -fn path_nodes<'py>( - start: Bound<'py, PyAny>, - end: Bound<'py, PyAny>, - downstream: &Bound<'py, PyAny>, -) -> PyResult>> { - let py: Python<'py> = downstream.py(); - let upstream: Bound<'py, PyDict> = inverted(downstream)?; - let none: Bound<'py, PyTuple> = PyTuple::empty(py); - let mut reachable: PyNodeSet<'py> = PyNodeSet(PySet::empty(py)?); - let mut path: PyNodeSet<'py> = PyNodeSet(PySet::empty(py)?); - let found: bool = directed_path_nodes( - (start, end), - ( - |current: &Bound<'py, PyAny>| neighbours(downstream, current, &none), - |current: &Bound<'py, PyAny>| neighbours(upstream.as_any(), current, &none), - ), - (&mut reachable, &mut path), - )?; - if !found { - return Ok(None); - } - Ok(Some(PyFrozenSet::new(py, path.0.iter())?)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(transitive_closure, module)?)?; - module.add_function(wrap_pyfunction!(transitive_closure_many, module)?)?; - module.add_function(wrap_pyfunction!(invert_edges, module)?)?; - module.add_function(wrap_pyfunction!(path_nodes, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs deleted file mode 100644 index 1b2c776580..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! Ordering edges attached audits and model hooks add, and the keys hooks and SQL refs name. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; - -use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; -use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_base::records::types::RecordId; -/// The model config keys whose entries are hooks, in lifecycle order. -const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; - -fn keyword_call<'py>( - callable: &Bound<'py, PyAny>, - pairs: &[(&str, Bound<'py, PyAny>)], -) -> PyResult> { - let keywords: Bound<'py, PyDict> = PyDict::new(callable.py()); - for (name, value) in pairs { - keywords.set_item(name, value)?; - } - callable.call((), Some(&keywords)) -} - -/// The Python hooks attached to one model, in lifecycle order without repeats. -#[pyfunction] -#[pyo3(signature = (*, model))] -pub(crate) fn model_python_hook_names<'py>( - py: Python<'py>, - model: &Bound<'py, PyAny>, -) -> PyResult> { - let entry_type: Bound<'_, PyAny> = record_class(py, RecordId::PythonHookEntry)?; - let values: Bound<'_, PyAny> = native_field(model, "config")?.getattr("values")?; - let mut names: Vec = Vec::new(); - for hook_key in MODEL_HOOK_KEYS { - let entries: Bound<'_, PyAny> = mapping_get(&values, hook_key)?; - if entries.is_instance(&entry_type)? { - names.push(native_field(&entries, "name")?.extract()?); - } else if entries.is_instance_of::() || entries.is_instance_of::() { - for entry in entries.try_iter()? { - let entry: Bound<'_, PyAny> = entry?; - if entry.is_instance(&entry_type)? { - names.push(native_field(&entry, "name")?.extract()?); - } - } - } - } - let mut unique: Vec = Vec::with_capacity(names.len()); - for name in names { - if !unique.contains(&name) { - unique.push(name); - } - } - PyTuple::new(py, unique) -} - -/// The compiled graph key a typed model, source or seed reference names. -#[pyfunction] -pub(crate) fn sql_ref_key<'py>( - py: Python<'py>, - reference: &Bound<'py, PyAny>, -) -> PyResult> { - let resource_type: Bound<'py, PyAny> = enum_from_value( - py, - RecordId::CompiledResourceType, - native_field(reference, "kind")?.getattr("value")?, - )?; - keyword_call( - &record_class(py, RecordId::CompiledObjectKey)?, - &[ - ("resource_type", resource_type), - ("name", native_field(reference, "name")?), - ], - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(model_python_hook_names, module)?)?; - module.add_function(wrap_pyfunction!(sql_ref_key, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs deleted file mode 100644 index 3b28885e9b..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! `CompiledObjectKey`s for native `(resource type, name)` pairs, converted once per pair. - -use std::cell::RefCell; -use std::collections::HashMap; - -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use sqlbuild_python_base::records::main::construct::record_class::record_class; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; - -/// One `CompiledObjectKey` per native pair, so shared keys convert once. -pub(crate) struct KeyCache<'py> { - key_type: Bound<'py, PyAny>, - resource_type: Bound<'py, PyAny>, - keys: RefCell>>, -} - -impl<'py> KeyCache<'py> { - pub(crate) fn new(py: Python<'py>) -> PyResult { - Ok(Self { - key_type: record_class(py, RecordId::CompiledObjectKey)?, - resource_type: record_class(py, RecordId::CompiledResourceType)?, - keys: RefCell::new(HashMap::new()), - }) - } - - pub(crate) fn py(&self) -> Python<'py> { - self.key_type.py() - } - - /// The `CompiledObjectKey` for one `(resource type, name)` pair. - pub(crate) fn key( - &self, - (resource_type, name): &(String, String), - ) -> PyResult> { - let pair = (resource_type.clone(), name.clone()); - if let Some(existing) = self.keys.borrow().get(&pair) { - return Ok(existing.bind(self.py()).clone()); - } - let created: Bound<'py, PyAny> = self - .key_type - .call1((self.resource_type.call1((resource_type,))?, name))?; - self.keys - .borrow_mut() - .insert(pair, created.clone().unbind()); - Ok(created) - } - - /// `CompiledObjectKey`s for `pairs`, in order. - pub(crate) fn keys(&self, pairs: &[(String, String)]) -> PyResult>> { - pairs.iter().map(|pair| self.key(pair)).collect() - } -} - -/// The native `(resource type, name)` pair of a `CompiledObjectKey`. -pub(crate) fn native_key(key: &Bound<'_, PyAny>) -> PyResult<(String, String)> { - Ok(( - native_field(key, "resource_type")?.str()?.extract()?, - native_field(key, "name")?.extract()?, - )) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs index 0db6458b40..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs @@ -1,11 +1 @@ -pub(crate) mod algorithms; -pub(crate) mod execution_gates; -pub(crate) mod getters; -pub(crate) mod keys; -pub(crate) mod node_set; -pub(crate) mod project_graph; -pub(crate) mod python_lifecycle; -pub(crate) mod python_node_graph; -pub(crate) mod python_selection; pub(crate) mod registration; -pub(crate) mod views; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/node_set.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/node_set.rs deleted file mode 100644 index 488de612b0..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/node_set.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! A Python set as the visited nodes of a native graph walk over Python keys. - -use pyo3::PyErr; -use pyo3::prelude::{Bound, PyAny}; -use pyo3::types::{PySet, PySetMethods}; -use sqlbuild_core::graph_walk::types::NodeSet; - -/// Python keys a walk visited, compared with Python hashing and equality. -pub(crate) struct PyNodeSet<'py>(pub(crate) Bound<'py, PySet>); - -impl<'py> NodeSet> for PyNodeSet<'py> { - type Error = PyErr; - - fn insert_new(&mut self, key: &Bound<'py, PyAny>) -> Result { - if self.0.contains(key)? { - return Ok(false); - } - self.0.add(key)?; - Ok(true) - } - - fn holds(&self, key: &Bound<'py, PyAny>) -> Result { - self.0.contains(key) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs deleted file mode 100644 index e8836bcbaa..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs +++ /dev/null @@ -1,340 +0,0 @@ -//! The compiled project graph and selector resolution, held natively for Python's graph users. - -use sqlbuild_python_base::positional_record; -use std::collections::HashMap; - -use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::sync::PyOnceLock; -use pyo3::{pyclass, pymethods}; -use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_analysis::graph::errors::SelectorError; -use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; -use sqlbuild_analysis::graph::main::build_resources::build_resources; -use sqlbuild_analysis::graph::main::edge_keys::edge_keys; -use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; -use sqlbuild_analysis::graph::main::match_selector::match_selector; -use sqlbuild_analysis::graph::main::model_layer_count::model_layer_count; -use sqlbuild_analysis::graph::main::resolve_selector_tokens::resolve_selector_tokens; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; -use sqlbuild_analysis::graph::main::unique_id_edges::unique_id_edges; -use sqlbuild_analysis::graph::models::{ - BuildResources, EdgeDirection, GraphIndexes, GraphResource, ProjectGraph, -}; - -use crate::bindings::_helpers::graph::views::lineage_graph_views; -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -positional_record! { - extract; - /// `(key, deps, tags, model folder)`. - pub(crate) struct ResourceRow { - pub(crate) key: ObjectKey, - pub(crate) deps: Vec, - pub(crate) tags: Vec, - pub(crate) folder: Option, - } -} -type Edges = Vec<(ObjectKey, Vec)>; -positional_record! { - extract, into; - /// `(names, upstream, downstream, tags, paths)` in Python's dict order. - struct IndexesRow { - names: Vec<(String, ObjectKey)>, - upstream: Edges, - downstream: Edges, - tags: Vec<(String, Vec)>, - paths: Vec<(ObjectKey, String)>, - } -} -/// `(unique id, neighbour unique ids)` in graph order. -type UniqueIdMap = Vec<(String, Vec)>; -/// `PlannerInputError`'s `(code, message, help)`. -type FailureRow = (&'static str, String, Option); -/// The keys, or the `PlannerInputError` Python raises instead. -type KeysOutcome = (Option>, Option); - -/// A project graph: lineage edges, tag, path and name indexes, and selector resolution. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NativeProjectGraph { - graph: ProjectGraph, -} - -/// Static compiled project graph held natively; dict views are built only when read. -#[pyclass(name = "ProjectGraph", module = "sqlbuild._native", frozen)] -pub(crate) struct CompiledProjectGraph { - project: Py, - native: Py, - views: PyOnceLock>, -} - -fn dependency_edges(key: &ObjectKey, deps: &[ObjectKey]) -> Vec<(ObjectKey, ObjectKey)> { - deps.iter().map(|dep| (dep.clone(), key.clone())).collect() -} - -fn outcome(result: Result, SelectorError>) -> KeysOutcome { - match result { - Ok(keys) => (Some(keys), None), - Err(error) => (None, Some(failure(error))), - } -} - -fn failure(error: SelectorError) -> FailureRow { - (error.code, error.message, error.help) -} - -impl NativeProjectGraph { - /// The graph this object wraps. - pub(crate) fn graph(&self) -> &ProjectGraph { - &self.graph - } - - /// Wrap a graph another native stage built, such as project assembly. - pub(crate) fn new(graph: ProjectGraph) -> Self { - Self { graph } - } -} - -impl CompiledProjectGraph { - /// The graph of `project` over its native lineage graph. - pub(crate) fn new(project: Py, native: Py) -> Self { - Self { - project, - native, - views: PyOnceLock::new(), - } - } - - /// The compiled project this graph indexes. - pub(crate) fn project<'py>(&self, py: Python<'py>) -> &Bound<'py, PyAny> { - self.project.bind(py) - } - - /// The native lineage graph. - pub(crate) fn native<'py>(&self, py: Python<'py>) -> &Bound<'py, NativeProjectGraph> { - self.native.bind(py) - } - - fn view(&self, py: Python<'_>, name: &str) -> PyResult> { - Ok(native_field(self.lineage_views(py)?.bind(py), name)?.unbind()) - } -} - -#[pymethods] -impl CompiledProjectGraph { - #[new] - #[pyo3(signature = (*, project, native))] - fn py_new(project: Py, native: Py) -> Self { - Self::new(project, native) - } - - #[getter(project)] - pub(crate) fn project_attribute(&self, py: Python<'_>) -> Py { - self.project.clone_ref(py) - } - - #[getter(native)] - pub(crate) fn native_attribute(&self, py: Python<'_>) -> Py { - self.native.clone_ref(py) - } - - /// Lineage edges and selector indexes, converted once. - #[getter] - pub(crate) fn lineage_views(&self, py: Python<'_>) -> PyResult> { - self.views - .get_or_try_init(py, || { - Ok(lineage_graph_views(py, self.native.bind(py).borrow())?.unbind()) - }) - .map(|views| views.clone_ref(py)) - } - - #[getter] - pub(crate) fn upstream_deps(&self, py: Python<'_>) -> PyResult> { - self.view(py, "upstream_deps") - } - - #[getter] - pub(crate) fn downstream_deps(&self, py: Python<'_>) -> PyResult> { - self.view(py, "downstream_deps") - } - - #[getter] - pub(crate) fn tag_index(&self, py: Python<'_>) -> PyResult> { - self.view(py, "tag_index") - } - - #[getter] - pub(crate) fn path_index(&self, py: Python<'_>) -> PyResult> { - self.view(py, "path_index") - } - - #[getter] - pub(crate) fn all_keys(&self, py: Python<'_>) -> PyResult> { - self.view(py, "all_keys") - } -} - -/// The project graph indexes of these resources, in project order. -pub(crate) fn project_graph(resources: Vec) -> PyResult { - compiler_guard(|| { - let resources: Vec = resources - .into_iter() - .map( - |ResourceRow { - key, - deps, - tags, - folder, - }| GraphResource { - key, - deps, - tags, - folder, - }, - ) - .collect(); - Ok(NativeProjectGraph::new(build_project_graph(&resources))) - }) -} - -#[pymethods] -impl NativeProjectGraph { - /// Build the indexes from resources in project order: models, sources, seeds, functions. - #[staticmethod] - fn from_resources(resources: Vec) -> PyResult { - project_graph(resources) - } - - /// Wrap indexes a caller already built, such as the planner's. - #[staticmethod] - fn from_indexes(indexes: IndexesRow) -> PyResult { - compiler_guard(|| { - let IndexesRow { - names, - upstream, - downstream, - tags, - paths, - } = indexes; - Ok(Self::new(graph_from_indexes(GraphIndexes { - names, - upstream, - downstream, - tags, - paths, - }))) - }) - } - - /// `(names, upstream, downstream, tags, paths)` for Python's `ProjectGraph`. - fn indexes(&self) -> IndexesRow { - let graph: &ProjectGraph = &self.graph; - IndexesRow { - names: graph.names.clone(), - upstream: graph.upstream.clone(), - downstream: graph.downstream.clone(), - tags: graph.tags.clone(), - paths: graph.paths.clone(), - } - } - - /// The key's direct upstream lineage keys, in authored dependency order. - pub(crate) fn upstream(&self, key: ObjectKey) -> Vec { - edge_keys(&self.graph, &key, EdgeDirection::Upstream).to_vec() - } - - /// The key's direct downstream keys, sorted by `(resource type, name)`. - fn downstream(&self, key: ObjectKey) -> Vec { - edge_keys(&self.graph, &key, EdgeDirection::Downstream).to_vec() - } - - /// Every selectable key, in project order with later same-name resources replacing earlier. - fn keys(&self) -> Vec { - self.graph - .names - .iter() - .map(|(_, key)| key.clone()) - .collect() - } - - /// `(selector name, key)` in project order. - fn names(&self) -> Vec<(String, ObjectKey)> { - self.graph.names.clone() - } - - /// Every lineage edge as `(dependency, dependent)`, in upstream order. - fn edges(&self) -> Vec<(ObjectKey, ObjectKey)> { - self.graph - .upstream - .iter() - .flat_map(|(key, deps)| dependency_edges(key, deps)) - .collect() - } - - /// The manifest's `(parent_map, child_map)` entries as dbt-style unique ids, in graph order. - fn unique_id_maps( - &self, - project_name: &str, - prefixes: HashMap, - ) -> (UniqueIdMap, UniqueIdMap) { - ( - unique_id_edges( - &self.graph, - EdgeDirection::Upstream, - project_name, - &prefixes, - ), - unique_id_edges( - &self.graph, - EdgeDirection::Downstream, - project_name, - &prefixes, - ), - ) - } - - /// The compile report's execution layer count over models. - fn model_layer_count(&self) -> usize { - model_layer_count(&self.graph) - } - - /// `--select` minus `--exclude`, plus the functions they need to build. - fn resolve(&self, select: Vec, exclude: Vec) -> PyResult { - compiler_guard(|| Ok(outcome(resolve_selectors(&self.graph, &select, &exclude)))) - } - - /// The keys selector tokens match, without build resources. - fn tokens(&self, selectors: Vec) -> PyResult { - compiler_guard(|| Ok(outcome(resolve_selector_tokens(&self.graph, &selectors)))) - } - - /// The keys one `kind:value` selector names before `+` expansion. - fn matched(&self, kind: &str, value: &str) -> PyResult { - compiler_guard(|| Ok(outcome(match_selector(&self.graph, kind, value)))) - } - - /// The selected keys plus the functions and seeds Python adds to build them. - fn build_resources( - &self, - selected: Vec, - include: (bool, bool, bool), - ) -> PyResult> { - compiler_guard(|| { - let (upstream_functions, upstream_seeds, downstream_functions) = include; - Ok(build_resources( - &self.graph, - &selected, - BuildResources { - upstream_functions, - upstream_seeds, - downstream_functions, - }, - )) - }) - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs index 971b8975e3..dbc2621dd3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs @@ -1,11 +1,18 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; +use sqlbuild_python_project::graph::main::nodes::register_algorithms::register_algorithms; +use sqlbuild_python_project::graph::main::nodes::register_execution_gates::register_execution_gates; +use sqlbuild_python_project::graph::main::project::register_project_graph::register_project_graph; +use sqlbuild_python_project::graph::main::project::register_views::register_views; +use sqlbuild_python_project::graph::main::python_nodes::register_python_lifecycle::register_python_lifecycle; +use sqlbuild_python_project::graph::main::python_nodes::register_python_node_graph::register_python_node_graph; +use sqlbuild_python_project::graph::main::python_nodes::register_python_selection::register_python_selection; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - crate::bindings::_helpers::graph::project_graph::register(module)?; - crate::bindings::_helpers::graph::algorithms::register(module)?; - crate::bindings::_helpers::graph::execution_gates::register(module)?; - crate::bindings::_helpers::graph::python_selection::register(module)?; - crate::bindings::_helpers::graph::python_lifecycle::register(module)?; - crate::bindings::_helpers::graph::python_node_graph::register(module)?; - crate::bindings::_helpers::graph::views::register(module) + register_project_graph(module)?; + register_algorithms(module)?; + register_execution_gates(module)?; + register_python_selection(module)?; + register_python_lifecycle(module)?; + register_python_node_graph(module)?; + register_views(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs deleted file mode 100644 index d76f28574d..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs +++ /dev/null @@ -1,206 +0,0 @@ -//! The native project graph's indexes as the Python dicts graph consumers read, and back. - -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; -use sqlbuild_analysis::graph::models::{GraphIndexes, ProjectGraph}; - -use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; -use sqlbuild_python_base::records::types::RecordId; - -/// The native graph's pairs for `keys`, in order. -#[pyfunction] -fn native_graph_keys(keys: &Bound<'_, PyAny>) -> PyResult> { - keys.try_iter()?.map(|key| native_key(&key?)).collect() -} - -/// `CompiledObjectKey`s for native graph key pairs. -#[pyfunction] -pub(crate) fn compiled_graph_keys<'py>( - py: Python<'py>, - keys: Vec, -) -> PyResult> { - let cache: KeyCache<'py> = KeyCache::new(py)?; - PyFrozenSet::new(py, cache.keys(&keys)?) -} - -/// Selector name to key, in project order, without the other graph views. -#[pyfunction] -pub(crate) fn lineage_graph_names<'py>( - py: Python<'py>, - graph: PyRef<'py, NativeProjectGraph>, -) -> PyResult> { - let cache: KeyCache<'py> = KeyCache::new(py)?; - let names: Bound<'py, PyDict> = PyDict::new(py); - for (name, key) in &graph.graph().names { - names.set_item(name, cache.key(key)?)?; - } - Ok(names) -} - -pub(crate) fn edge_dict<'py>( - cache: &KeyCache<'py>, - edges: &[(ObjectKey, Vec)], -) -> PyResult> { - let py: Python<'py> = cache.py(); - let dict: Bound<'py, PyDict> = PyDict::new(py); - for (key, deps) in edges { - dict.set_item(cache.key(key)?, PyTuple::new(py, cache.keys(deps)?)?)?; - } - Ok(dict) -} - -/// The native graph's indexes as `LineageGraphViews` dicts, in project order. -fn graph_views<'py>(py: Python<'py>, graph: &ProjectGraph) -> PyResult> { - let cache: KeyCache<'py> = KeyCache::new(py)?; - let tags: Bound<'py, PyDict> = PyDict::new(py); - for (tag, keys) in &graph.tags { - tags.set_item(tag, PyFrozenSet::new(py, cache.keys(keys)?)?)?; - } - let paths: Bound<'py, PyDict> = PyDict::new(py); - for (key, folder) in &graph.paths { - paths.set_item(cache.key(key)?, folder)?; - } - let names: Bound<'py, PyDict> = PyDict::new(py); - for (name, key) in &graph.names { - names.set_item(name, cache.key(key)?)?; - } - let keywords: Bound<'py, PyDict> = PyDict::new(py); - keywords.set_item("upstream_deps", edge_dict(&cache, &graph.upstream)?)?; - keywords.set_item("downstream_deps", edge_dict(&cache, &graph.downstream)?)?; - keywords.set_item("tag_index", tags)?; - keywords.set_item("path_index", paths)?; - keywords.set_item("all_keys", names)?; - new_record(py, RecordId::LineageGraphViews, (), Some(&keywords)) -} - -/// The graph's lineage edges and selector indexes as dicts, in project order. -#[pyfunction] -pub(crate) fn lineage_graph_views<'py>( - py: Python<'py>, - graph: PyRef<'py, NativeProjectGraph>, -) -> PyResult> { - graph_views(py, graph.graph()) -} - -/// The project's lineage edges and selector indexes, from its native graph. -#[pyfunction] -pub(crate) fn project_lineage_views<'py>( - py: Python<'py>, - project: &Bound<'py, PyAny>, -) -> PyResult> { - let graph: PyRef<'py, NativeProjectGraph> = - native_field(project, "lineage_graph")?.extract()?; - graph_views(py, graph.graph()) -} - -fn native_edges(mapping: &Bound<'_, PyAny>) -> PyResult)>> { - let mut edges: Vec<(ObjectKey, Vec)> = Vec::new(); - for item in mapping_items(mapping)?.try_iter()? { - let (key, deps): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; - edges.push((native_key(&key)?, native_graph_keys(&deps)?)); - } - Ok(edges) -} - -fn graph_from_dicts(indexes: GraphIndexes) -> PyResult { - compiler_guard(|| Ok(NativeProjectGraph::new(graph_from_indexes(indexes)))) -} - -/// A native graph over these indexes, keeping their order. -#[pyfunction] -#[pyo3(signature = (**request))] -fn native_graph_from_views(request: Option<&Bound<'_, PyDict>>) -> PyResult { - let Some(request) = request else { - return Err(pyo3::exceptions::PyTypeError::new_err( - "native_graph_from_views() missing keyword arguments", - )); - }; - let field = |name: &str| -> PyResult> { - request.get_item(name)?.ok_or_else(|| { - pyo3::exceptions::PyTypeError::new_err(format!( - "native_graph_from_views() missing keyword argument '{name}'" - )) - }) - }; - graph_from_view_mappings(GraphViewMappings { - all_keys: &field("all_keys")?, - upstream: &field("upstream")?, - downstream: &field("downstream")?, - tag_index: &field("tag_index")?, - path_index: &field("path_index")?, - }) -} - -/// Python's selector name, edge, tag and path mappings. -pub(crate) struct GraphViewMappings<'a, 'py> { - pub(crate) all_keys: &'a Bound<'py, PyAny>, - pub(crate) upstream: &'a Bound<'py, PyAny>, - pub(crate) downstream: &'a Bound<'py, PyAny>, - pub(crate) tag_index: &'a Bound<'py, PyAny>, - pub(crate) path_index: &'a Bound<'py, PyAny>, -} - -/// A native graph over Python's index mappings, keeping their order. -pub(crate) fn graph_from_view_mappings( - mappings: GraphViewMappings<'_, '_>, -) -> PyResult { - let mut names: Vec<(String, ObjectKey)> = Vec::new(); - for item in mapping_items(mappings.all_keys)?.try_iter()? { - let (name, key): (String, Bound<'_, PyAny>) = item?.extract()?; - names.push((name, native_key(&key)?)); - } - let mut tags: Vec<(String, Vec)> = Vec::new(); - for item in mapping_items(mappings.tag_index)?.try_iter()? { - let (tag, keys): (String, Bound<'_, PyAny>) = item?.extract()?; - tags.push((tag, native_graph_keys(&keys)?)); - } - let mut paths: Vec<(ObjectKey, String)> = Vec::new(); - for item in mapping_items(mappings.path_index)?.try_iter()? { - let (key, folder): (Bound<'_, PyAny>, String) = item?.extract()?; - paths.push((native_key(&key)?, folder)); - } - graph_from_dicts(GraphIndexes { - names, - upstream: native_edges(mappings.upstream)?, - downstream: native_edges(mappings.downstream)?, - tags, - paths, - }) -} - -/// A native graph holding these edges in their order, with no selector indexes. -#[pyfunction] -#[pyo3(signature = (*, upstream, downstream))] -pub(crate) fn native_graph_from_edges( - upstream: &Bound<'_, PyAny>, - downstream: &Bound<'_, PyAny>, -) -> PyResult { - graph_from_dicts(GraphIndexes { - names: Vec::new(), - upstream: native_edges(upstream)?, - downstream: native_edges(downstream)?, - tags: Vec::new(), - paths: Vec::new(), - }) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(native_graph_keys, module)?)?; - module.add_function(wrap_pyfunction!(compiled_graph_keys, module)?)?; - module.add_function(wrap_pyfunction!(lineage_graph_names, module)?)?; - module.add_function(wrap_pyfunction!(lineage_graph_views, module)?)?; - module.add_function(wrap_pyfunction!(project_lineage_views, module)?)?; - module.add_function(wrap_pyfunction!(native_graph_from_views, module)?)?; - module.add_function(wrap_pyfunction!(native_graph_from_edges, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs index 5740df9c2a..9a364b646f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs @@ -5,7 +5,6 @@ use crate::bindings::_helpers::analysis::binding_locations::{ }; use crate::bindings::_helpers::analysis_session::outcome_objects::dynamic_contract_proof; use crate::bindings::_helpers::analysis_session::outcome_objects::session_outcome_analysis; -use crate::bindings::models::Positional; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; @@ -16,6 +15,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::analysis::models::Positional; type Object<'py> = Bound<'py, PyAny>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs index de4ef660c3..a12c692d64 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs @@ -14,23 +14,10 @@ use sqlbuild_rules::lint_text::constants::{DEFAULT_LINE_WIDTH, DEFAULT_MAX_DESCR use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; use sqlbuild_rules::lint_text::models::FormatSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditBlockRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditFileRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::DiagnosticSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::FunctionRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::FunctionSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::LoaderRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::ScenarioRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::SourceRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::TestBlockRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::TestCaseRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::TestFileRow; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::input_loops::resource_attachment::{ attach_audits, attach_functions, attach_sources, attach_sql_tests, }; use crate::bindings::_helpers::input_loops::resource_diagnostics::resource_diagnostics; -use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_CTE_SCAN_ALGORITHM; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -44,6 +31,19 @@ use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_project::models::AuditBlockRow; +use sqlbuild_python_project::compiled_project::models::AuditFileRow; +use sqlbuild_python_project::compiled_project::models::DiagnosticSettings; +use sqlbuild_python_project::compiled_project::models::FunctionRow; +use sqlbuild_python_project::compiled_project::models::FunctionSettings; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_project::models::ScenarioRow; +use sqlbuild_python_project::compiled_project::models::TestBlockRow; +use sqlbuild_python_project::compiled_project::models::TestFileRow; +use sqlbuild_python_project::compiled_project::types::LoaderRow; +use sqlbuild_python_project::compiled_project::types::SourceRow; +use sqlbuild_python_project::compiled_project::types::TestCaseRow; +use sqlbuild_python_project::sql_test_glue::constants::SQL_TEST_CTE_SCAN_ALGORITHM; const YAML_FILE_SUFFIXES: [&str; 2] = [".yml", ".yaml"]; const PATHLIB_MODULE: &str = "pathlib"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs index 7c03d6d5f0..f433c73f53 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs @@ -22,10 +22,6 @@ use sqlbuild_model_config::errors::ConfigError; use sqlbuild_model_config::header_metadata::main::parse_measurement_policy::parse_measurement_policy; use sqlbuild_sqltext::sql_references::models::SqlReference; -use crate::bindings::_helpers::compiled_project::attachment_inputs::{ - AuditBlockRow, AuditFileRow, AuditSettings, -}; -use crate::bindings::_helpers::compiled_views::audit_views::{AttachedAudit, AuditHandles}; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, compile_error, helped_compile_error, }; @@ -42,11 +38,15 @@ use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::compile_context::main::delta::record_source_reads::record_source_reads; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::main::headers::thresholds::thresholds; -use sqlbuild_python_front::model_config::models::AuditInstance; -use sqlbuild_python_front::model_config::models::HeaderAudits; use sqlbuild_python_front::model_config::models::HeaderColumns; use sqlbuild_python_front::model_config::models::NativeModelValidator; use sqlbuild_python_front::model_config::models::PyNode; +use sqlbuild_python_front::model_config::models::{AuditInstance, HeaderAudits}; +use sqlbuild_python_project::compiled_project::models::{ + AuditBlockRow, AuditFileRow, AuditSettings, +}; +use sqlbuild_python_project::resource_views::models::AuditHandles; +use sqlbuild_python_project::resource_views::types::AttachedAudit; const THRESHOLDS_KEY: &str = "thresholds"; const MINIMUM_SAMPLES_KEY: &str = "minimum_samples"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs index 19050d2648..b0fa440e2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs @@ -28,13 +28,6 @@ use sqlbuild_attachments::functions::models::{ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; -use crate::bindings::_helpers::analysis_session::query_columns::query_columns_error; -use crate::bindings::_helpers::compiled_project::attachment_inputs::{ - FunctionRow, FunctionSettings, -}; -use crate::bindings::_helpers::compiled_views::function_views::{ - AttachedFunction, FunctionHandles, -}; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, compile_error, }; @@ -50,6 +43,10 @@ use sqlbuild_project::types::NamedTypeRow; use sqlbuild_python_base::compile_context::main::delta::record_source_reads::record_source_reads; use sqlbuild_python_front::attachments::main::parsed_header::parsed_header; use sqlbuild_python_front::model_config::models::NativeModelValidator; +use sqlbuild_python_project::analysis_session::main::query_columns_error::query_columns_error; +use sqlbuild_python_project::compiled_project::models::{FunctionRow, FunctionSettings}; +use sqlbuild_python_project::resource_views::models::FunctionHandles; +use sqlbuild_python_project::resource_views::types::AttachedFunction; const PRESERVE_TARGET_VALUE: &str = "preserve"; const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs index b3f4d41d67..25ff2af941 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs @@ -15,6 +15,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::project::declaration_rows_function::declaration_rows_function; /// The materializations a model's storage policies are inherited for, besides none. const TABLE_BACKED_MATERIALIZATIONS: [&str; 3] = ["TABLE", "INCREMENTAL", "SNAPSHOT"]; @@ -378,11 +379,7 @@ pub(crate) fn model_loop_classes<'py>( py, vec![( "declaration_rows", - pyo3::wrap_pyfunction!( - crate::bindings::_helpers::compiled_views::records::declaration_rows, - py - )? - .into_any(), + declaration_rows_function(py)?.into_any(), )], )?; dict( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index 962dbc097b..f086f52c52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -31,23 +31,6 @@ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_ use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::compiled_project::attachment_inputs::AttachedSqlTests; -use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditFileRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::FunctionRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::FunctionSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::ScenarioRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::SourceRow; -use crate::bindings::_helpers::compiled_project::attachment_inputs::SourceSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::SqlTestSettings; -use crate::bindings::_helpers::compiled_project::attachment_inputs::TestFileRow; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::audit_views::AttachedAudit; -use crate::bindings::_helpers::compiled_views::function_views::AttachedFunction; -use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_views::source_views::{ - AttachedSource, NativeSourceInput, retain_sources, -}; use crate::bindings::_helpers::input_loops::audit_loop::{AttachmentOwner, attached_audits}; use crate::bindings::_helpers::input_loops::function_loop::{ attached_functions, checked_table_function_arities, @@ -95,8 +78,7 @@ use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::cache::models::NativeParseStore; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; -use sqlbuild_python_front::discovery::models::ModelFileTexts; -use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::discovery::models::{ModelFileTexts, NativeModelFile}; use sqlbuild_python_front::model_config::main::config::raised_config_exception::raised_config_exception; use sqlbuild_python_front::model_config::models::ConfigClasses; use sqlbuild_python_front::model_config::models::ContractClasses; @@ -104,18 +86,34 @@ use sqlbuild_python_front::model_config::models::NativeModelValidator; use sqlbuild_python_front::render::main::calls::span_row::span_row; use sqlbuild_python_front::render::main::declarations::expand_declarations::expand_declarations; use sqlbuild_python_front::render::models::ConstantRendering; -use sqlbuild_python_front::render::models::DeclarationExpansion; use sqlbuild_python_front::render::models::DeclarationExpansionClasses; -use sqlbuild_python_front::render::models::ExpandedDeclarations; -use sqlbuild_python_front::render::models::ExpansionConsumer; -use sqlbuild_python_front::render::models::SqlCallOrigin; -use sqlbuild_python_front::render::models::SqlMacroSource; +use sqlbuild_python_front::render::models::{ + DeclarationExpansion, ExpandedDeclarations, ExpansionConsumer, +}; +use sqlbuild_python_front::render::models::{SqlCallOrigin, SqlMacroSource}; use sqlbuild_python_front::render::types::SpanRow; use sqlbuild_python_front::sqltext::main::declaration_scan::declaration_scan; use sqlbuild_python_front::sqltext::main::interpolation_batch::interpolation_batch; use sqlbuild_python_front::sqltext::models::InterpolationRow as SqlInterpolationRow; use sqlbuild_python_front::sqltext::models::InterpolationSources; use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; +use sqlbuild_python_project::compiled_project::models::AuditFileRow; +use sqlbuild_python_project::compiled_project::models::AuditSettings; +use sqlbuild_python_project::compiled_project::models::FunctionRow; +use sqlbuild_python_project::compiled_project::models::FunctionSettings; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_project::models::ScenarioRow; +use sqlbuild_python_project::compiled_project::models::SqlTestSettings; +use sqlbuild_python_project::compiled_project::models::TestFileRow; +use sqlbuild_python_project::compiled_project::types::AttachedSqlTests; +use sqlbuild_python_project::compiled_project::types::SourceRow; +use sqlbuild_python_project::compiled_project::types::SourceSettings; +use sqlbuild_python_project::compiled_views::models::NativeModelInput; +use sqlbuild_python_project::resource_views::main::source_views::retain_sources::retain_sources; +use sqlbuild_python_project::resource_views::models::NativeSourceInput; +use sqlbuild_python_project::resource_views::types::AttachedAudit; +use sqlbuild_python_project::resource_views::types::AttachedFunction; +use sqlbuild_python_project::resource_views::types::AttachedSource; const MATERIALIZED_KEY: &str = "materialized"; const CURSOR_KEY: &str = "cursor"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs index 502891c5c4..76248a4473 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_config.rs @@ -18,11 +18,11 @@ use sqlbuild_python_front::discovery::models::NativeModelFile; use sqlbuild_python_front::model_config::main::config::carried_node::carried_node; use sqlbuild_python_front::model_config::main::config::config_fields::config_fields; use sqlbuild_python_front::model_config::main::config::view::view; -use sqlbuild_python_front::model_config::models::BuiltConfig; -use sqlbuild_python_front::model_config::models::ConfigBuild; use sqlbuild_python_front::model_config::models::ConfigClasses; -use sqlbuild_python_front::model_config::models::NativeModelConfigBuilder; use sqlbuild_python_front::model_config::models::PyNode; +use sqlbuild_python_front::model_config::models::{ + BuiltConfig, ConfigBuild, NativeModelConfigBuilder, +}; use sqlbuild_python_front::model_config::types::BuiltConfigValues; const COLUMNS_KEY: &str = "columns"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs index ed82993a64..b1e5f7f3f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs @@ -13,8 +13,7 @@ use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_model_config::header_metadata::models::ParsedAudit; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::main::config::config_node::config_node; -use sqlbuild_python_front::model_config::models::HeaderColumn; -use sqlbuild_python_front::model_config::models::HeaderColumns; +use sqlbuild_python_front::model_config::models::{HeaderColumn, HeaderColumns}; use sqlbuild_python_front::model_config::types::PyConfigNode; const CASES_ATTRIBUTE: &str = "cases"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs index 0becc419a9..c342a1607f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_header.rs @@ -17,7 +17,6 @@ use sqlbuild_model_config::header_metadata::models::ParsedColumn; use sqlbuild_model_config::model_validation::models::ColumnMigration; use sqlbuild_model_config::types::NodeKind; -use crate::bindings::_helpers::compiled_project::schema_facts::family_fact; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, compile_error, config_error, }; @@ -41,11 +40,10 @@ use sqlbuild_python_front::model_config::main::headers::native_schema_column::na use sqlbuild_python_front::model_config::main::headers::parsed_audit_identity::parsed_audit_identity; use sqlbuild_python_front::model_config::main::model_nodes::column_migration_error::column_migration_error; use sqlbuild_python_front::model_config::models::HeaderAudits; -use sqlbuild_python_front::model_config::models::HeaderColumn; -use sqlbuild_python_front::model_config::models::HeaderColumns; use sqlbuild_python_front::model_config::models::HeaderFamilies; -use sqlbuild_python_front::model_config::models::NativeColumnSpec; +use sqlbuild_python_front::model_config::models::{HeaderColumn, HeaderColumns, NativeColumnSpec}; use sqlbuild_python_front::model_config::types::HeaderValue; +use sqlbuild_python_project::compiled_project::main::family_fact::family_fact; const DESCRIPTION_KEY: &str = "description"; const COLUMNS_KEY: &str = "columns"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs index 5ac32c69f0..73a36178f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_hooks.rs @@ -38,11 +38,11 @@ use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::attachments::main::argument_pairs::argument_pairs; use sqlbuild_python_front::render::main::declarations::expand_declarations::expand_declarations; -use sqlbuild_python_front::render::models::DeclarationExpansion; use sqlbuild_python_front::render::models::DeclarationExpansionClasses; -use sqlbuild_python_front::render::models::ExpandedDeclarations; -use sqlbuild_python_front::render::models::ExpansionConsumer; use sqlbuild_python_front::render::models::SqlMacroSource; +use sqlbuild_python_front::render::models::{ + DeclarationExpansion, ExpandedDeclarations, ExpansionConsumer, +}; use sqlbuild_python_front::sqltext::main::declaration_scan::declaration_scan; use sqlbuild_python_front::sqltext::main::interpolated::interpolated; use sqlbuild_python_front::sqltext::models::InterpolationSources; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs index d75cfcf5b0..add77b1fbe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs @@ -13,15 +13,6 @@ use sqlbuild_analysis::compiled_project::models::{ use sqlbuild_analysis::sql_macros::models::TypedCall; use sqlbuild_discovery::model_files::models::HeaderFacts; -use crate::bindings::_helpers::compiled_project::project::{ - CONTRACT_KEY, INCREMENTAL_MODE_KEY, materialization, text_value, -}; -use crate::bindings::_helpers::compiled_project::project::{ - DeclarationPy, config_values, declarations, -}; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - InputProjections, ModelInputHandles, ModelInputRecord, SchemaHandles, SchemaRecord, -}; use crate::bindings::_helpers::input_loops::model_loop_header::AttachedHeader; use crate::bindings::_helpers::input_loops::model_loop_inputs::{ HOOK_KEYS, reference_fact, stripped_values, @@ -30,8 +21,7 @@ use sqlbuild_project::models::ExpansionRecord; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; -use sqlbuild_python_front::discovery::models::ModelFileTexts; -use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_front::discovery::models::{ModelFileTexts, NativeModelFile}; use sqlbuild_python_front::model_config::main::config::carried_node::carried_node; use sqlbuild_python_front::model_config::main::config::retain_values::retain_values; use sqlbuild_python_front::model_config::main::config::typed_values::typed_values; @@ -41,6 +31,16 @@ use sqlbuild_python_front::render::models::ExpandedDeclarations; use sqlbuild_python_front::render::models::ExpansionOutput; use sqlbuild_python_front::render::types::SpanRow; use sqlbuild_python_front::sqltext::models::ScannedReference; +use sqlbuild_python_project::compiled_project::constants::{CONTRACT_KEY, INCREMENTAL_MODE_KEY}; +use sqlbuild_python_project::compiled_project::main::config_values::config_values; +use sqlbuild_python_project::compiled_project::main::declarations::declarations; +use sqlbuild_python_project::compiled_project::main::materialization::materialization; +use sqlbuild_python_project::compiled_project::main::text_value::text_value; +use sqlbuild_python_project::compiled_project::models::DeclarationPy; +use sqlbuild_python_project::compiled_views::models::{ + InputProjections, ModelInputHandles, SchemaHandles, +}; +use sqlbuild_python_project::compiled_views::types::{ModelInputRecord, SchemaRecord}; const HEADER_KEYS_ATTRIBUTE: &str = "model_header_keys"; const LAYER_SCHEMA_ATTRIBUTE: &str = "layer_schema"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs index 99dff69f7d..7db7b0df27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_macros.rs @@ -17,9 +17,7 @@ use crate::bindings::_helpers::input_loops::model_loop_inputs::reference_row; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::render::main::expansion::expanded::expanded; -use sqlbuild_python_front::render::models::ExpansionOutput; -use sqlbuild_python_front::render::models::NativeMacroExpansion; -use sqlbuild_python_front::render::models::StateInput; +use sqlbuild_python_front::render::models::{ExpansionOutput, NativeMacroExpansion, StateInput}; use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; /// Fields of `_VisibleModelDeclarations` a `DeclarationResolutionContext` is built from. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs index 7e6418483a..c3618fe353 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_opt_out.rs @@ -22,10 +22,10 @@ use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_core::constants::PANIC_MESSAGE; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::input_loops::model_loop_prepass::spawned; use sqlbuild_python_base::cache::main::parse_key::parse_key; use sqlbuild_python_base::cache::models::NativeParseStore; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, compile_error, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs index 0a25bfd463..51a18e67a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_schema.rs @@ -5,11 +5,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; -use crate::bindings::_helpers::compiled_project::schema_facts::python_column_fact; use crate::bindings::_helpers::input_loops::model_loop::{LoopContext, compile_error}; use crate::bindings::_helpers::input_loops::model_loop_inputs::present; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::models::HeaderColumn; +use sqlbuild_python_project::compiled_project::main::python_column_fact::python_column_fact; const MODEL_SCHEMA_KEY: &str = "model_schema"; const NAME_FIELD: &str = "name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs index b18d7efcaa..65d017392a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/registration.rs @@ -1,10 +1,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::compiled_project::attachment_inputs; use crate::bindings::_helpers::input_loops::resource_attachment::RESOURCE_ATTACHMENT; +use sqlbuild_python_project::compiled_project::main::install::install; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - attachment_inputs::install(RESOURCE_ATTACHMENT); + install(RESOURCE_ATTACHMENT); crate::bindings::_helpers::input_loops::model_loop::register(module)?; crate::bindings::_helpers::input_loops::attachment::register(module)?; crate::bindings::_helpers::input_loops::expanded_offsets::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs index 9b584bb8e4..55d0611ed3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_attachment.rs @@ -4,17 +4,6 @@ use std::collections::HashMap; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use crate::bindings::_helpers::compiled_project::attachment_inputs::{ - AttachedSqlTests, AuditFileRow, AuditSettings, FunctionRow, FunctionSettings, - ResourceAttachment, ScenarioRow, SourceRow, SourceSettings, SqlTestSettings, TestFileRow, -}; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::audit_views::NativeAuditInput; -use crate::bindings::_helpers::compiled_views::function_views::{ - AttachedFunction, NativeFunctionInput, -}; -use crate::bindings::_helpers::compiled_views::seed_views::{SeedParts, SeedViews}; -use crate::bindings::_helpers::compiled_views::source_views::{AttachedSource, NativeSourceInput}; use crate::bindings::_helpers::input_loops::audit_loop::{ AttachmentOwner, model_owner, resource_owner, }; @@ -22,6 +11,21 @@ use crate::bindings::_helpers::input_loops::model_loop::NativeModelLoop; use crate::bindings::_helpers::input_loops::resource_diagnostics::resource_diagnostics; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_project::models::{ + AuditFileRow, AuditSettings, FunctionRow, FunctionSettings, ResourceAttachment, ScenarioRow, + SqlTestSettings, TestFileRow, +}; +use sqlbuild_python_project::compiled_project::types::{ + AttachedSqlTests, SourceRow, SourceSettings, +}; +use sqlbuild_python_project::resource_views::models::NativeAuditInput; +use sqlbuild_python_project::resource_views::models::NativeFunctionInput; +use sqlbuild_python_project::resource_views::models::NativeSourceInput; +use sqlbuild_python_project::resource_views::models::SeedParts; +use sqlbuild_python_project::resource_views::types::AttachedFunction; +use sqlbuild_python_project::resource_views::types::AttachedSource; +use sqlbuild_python_project::resource_views::types::SeedViews; const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; const SOURCE_OWNER: &str = "source"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index 4ea914fa10..8678d572a6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -22,12 +22,6 @@ use sqlbuild_pysource::definition_locations::main::decorator_location::decorator use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_count; use sqlbuild_sqltext::compiler::main::model_header_entry_span::model_header_entry_span; -use crate::bindings::_helpers::compiled_project::attachment_inputs::{ - DiagnosticSettings, HookDescriptionRow, LoaderRow, PythonResourceRow, ResourceDiagnostics, - ScenarioDescriptionRow, -}; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::seed_views::{SeedParts, SeedViews}; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::compile_context::main::delta::capture::capture; use sqlbuild_python_base::compile_context::main::delta::capturing::capturing; @@ -35,6 +29,13 @@ use sqlbuild_python_base::compile_context::main::delta::injection::injection; use sqlbuild_python_base::compile_context::models::DeltaInjection; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::discovery::models::NativeModelFile; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_project::models::{ + DiagnosticSettings, HookDescriptionRow, PythonResourceRow, ScenarioDescriptionRow, +}; +use sqlbuild_python_project::compiled_project::types::{LoaderRow, ResourceDiagnostics}; +use sqlbuild_python_project::resource_views::models::SeedParts; +use sqlbuild_python_project::resource_views::types::SeedViews; const MISSING_DESCRIPTION_CODE: &str = "P010"; const DESCRIPTION_LENGTH_CODE: &str = "P027"; @@ -402,7 +403,7 @@ fn docstring_text(description: &Bound<'_, PyString>) -> String { /// A source's P010, falling back to its loader's description and naming the loader in help. fn source_description<'py>( diagnostics: &Diagnostics<'_, 'py>, - source: &crate::bindings::_helpers::compiled_views::source_views::SourceRecord, + source: &sqlbuild_python_project::resource_views::types::SourceRecord, ) -> PyResult>> { let py: Python<'py> = diagnostics.py; let attached = source.attached(); @@ -449,7 +450,7 @@ fn source_description<'py>( /// A SQL or Python function's P010, located at its header or `@udf` decorator. fn function_description<'py>( diagnostics: &Diagnostics<'_, 'py>, - function: &crate::bindings::_helpers::compiled_views::function_views::FunctionRecord, + function: &sqlbuild_python_project::resource_views::types::FunctionRecord, ) -> PyResult>> { let py: Python<'py> = diagnostics.py; let attached = &function.attached; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs index 3745d12287..5147d83a59 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/scenario_loop.rs @@ -14,7 +14,6 @@ use sqlbuild_attachments::test_targets::main::scenario_source_violation::scenari use sqlbuild_attachments::test_targets::models::ScenarioCteSources; use sqlbuild_sqltext::sql_references::models::SqlReference; -use crate::bindings::_helpers::compiled_project::attachment_inputs::ScenarioRow; use crate::bindings::_helpers::input_loops::model_loop::compile_error; use crate::bindings::_helpers::input_loops::model_loop_calls::origin_references; use crate::bindings::_helpers::input_loops::source_loop::{ @@ -24,6 +23,7 @@ use crate::bindings::_helpers::input_loops::sql_test_loop::{ FileOrigin, TestScope, assertion_targets, checked_body, source_names, unique, }; use sqlbuild_python_base::compile_context::main::delta::record_test_source_reads::record_test_source_reads; +use sqlbuild_python_project::compiled_project::models::ScenarioRow; impl<'py> TestScope<'_, 'py> { /// Every scenario expanded, classified and checked, in discovery order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs index fb42efb29e..484c2e0534 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs @@ -7,10 +7,6 @@ use sqlbuild_attachments::cursor_intrinsics::main::intrinsic_free::intrinsic_fre use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; use sqlbuild_model_config::templates::models::TemplateOptions; -use crate::bindings::_helpers::compiled_project::attachment_inputs::{SourceRow, SourceSettings}; -use crate::bindings::_helpers::compiled_views::source_views::{ - AttachedSource, SourceHandles, replaced, -}; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, compile_error, record_reads, }; @@ -26,19 +22,21 @@ use sqlbuild_python_front::model_config::main::config::expanded::expanded; use sqlbuild_python_front::model_config::main::config::native_config_error::native_config_error; use sqlbuild_python_front::model_config::main::config::raised_config_exception::raised_config_exception; use sqlbuild_python_front::model_config::main::config::template_error::template_error; -use sqlbuild_python_front::model_config::models::PythonHost; -use sqlbuild_python_front::model_config::models::Stop; -use sqlbuild_python_front::model_config::models::TemplateSources; +use sqlbuild_python_front::model_config::models::{PythonHost, Stop, TemplateSources}; use sqlbuild_python_front::render::main::declarations::expand_declarations::expand_declarations; -use sqlbuild_python_front::render::models::DeclarationExpansion; use sqlbuild_python_front::render::models::DeclarationExpansionClasses; -use sqlbuild_python_front::render::models::ExpandedDeclarations; -use sqlbuild_python_front::render::models::ExpansionConsumer; use sqlbuild_python_front::render::models::SqlMacroSource; +use sqlbuild_python_front::render::models::{ + DeclarationExpansion, ExpandedDeclarations, ExpansionConsumer, +}; use sqlbuild_python_front::scopes::models::NativeDeclarationContexts; use sqlbuild_python_front::sqltext::main::declaration_scan::declaration_scan; use sqlbuild_python_front::sqltext::main::interpolated::interpolated; use sqlbuild_python_front::sqltext::models::InterpolationSources; +use sqlbuild_python_project::compiled_project::types::{SourceRow, SourceSettings}; +use sqlbuild_python_project::resource_views::main::source_views::replaced::replaced; +use sqlbuild_python_project::resource_views::models::SourceHandles; +use sqlbuild_python_project::resource_views::types::AttachedSource; const MACRO_TOKEN: char = '@'; const NATIVE_CONTEXTS_ATTRIBUTE: &str = "native_contexts"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs index 5a75d64f90..9f0b8e0f3e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/sql_test_loop.rs @@ -35,10 +35,6 @@ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_ use sqlbuild_sqltext::sql_references::models::{ReferenceExtraction, SqlReference}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::bindings::_helpers::compiled_project::attachment_inputs::{ - SqlTestSettings, TestBlockRow, TestCaseRow, TestFileRow, -}; -use crate::bindings::_helpers::compiled_views::source_views::replaced; use crate::bindings::_helpers::input_loops::model_loop::{ LoopContext, NativeModelLoop, compile_error, helped_compile_error, }; @@ -52,6 +48,11 @@ use sqlbuild_python_base::cache::models::SqlTestScanStore; use sqlbuild_python_base::compile_context::main::delta::record_test_source_reads::record_test_source_reads; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::render::main::sql_macros::without_with_list_calls::without_with_list_calls; +use sqlbuild_python_project::compiled_project::models::{ + SqlTestSettings, TestBlockRow, TestFileRow, +}; +use sqlbuild_python_project::compiled_project::types::TestCaseRow; +use sqlbuild_python_project::resource_views::main::source_views::replaced::replaced; const MODEL_MODE: &str = "model"; const MACRO_MODE: &str = "macro"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs index 97e2c26b3b..b36f00df8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_classification.rs @@ -26,8 +26,8 @@ use crate::bindings::_helpers::janitor::janitor_keys::{ relation_age_timestamp, sorted_schemas, text, timestamp_fields, tuple, utc_datetime, }; use crate::bindings::_helpers::planner_scenario::artifacts::is_scenario_artifact_physical_name; -use crate::bindings::_helpers::relation_naming::fitting::shared_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::relation_naming::main::shared_error::shared_error; /// Protection rules for one plan. pub(crate) struct Protection { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 96bee69a6d..ac79a21cd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -38,7 +38,6 @@ use crate::bindings::_helpers::janitor::janitor_phases::{ }; use crate::bindings::_helpers::janitor::janitor_plan::{janitor_plan_text, janitor_result_counts}; use crate::bindings::_helpers::janitor::janitor_progress::JanitorProgressPrinter; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::scenario_command::setup::stream_isatty; use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -47,6 +46,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; +use sqlbuild_python_project::planner_phases::main::object::object; const COMMAND_NAME: &str = "janitor"; /// The code of a cleanup that needs a confirmation stdin cannot give. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index b01d11d52c..c66981ddf0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -9,7 +9,6 @@ use sqlbuild_analysis::janitor_planning::constants::UNTRACKED_QUERY_DIFF_REASON; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; -use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use crate::bindings::_helpers::janitor::janitor_keys::{ Folded, Key, SchemaKey, dedupe_schema_keys, folded_schema, integer, optional, record, sorted_schemas, text, tuple, @@ -18,6 +17,7 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::replaced::replaced; +use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; const QUERY_ARTIFACT_LIFECYCLE: ExecutorHost = ExecutorHost::QueryDiffArtifactLifecycle; const INSPECTION_LIST: AdapterHost = AdapterHost::ListRelationsForInspection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs index 4da294beb4..82be27aa65 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs @@ -9,7 +9,6 @@ use sqlbuild_analysis::janitor_planning::main::old_name_request_matches::old_nam use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::janitor::janitor_facts::{ SchemaRelations, Warehouse, model_destinations, }; @@ -21,6 +20,7 @@ use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_front::state_rows::main::migrations::read_histories::read_histories as read_view_histories; use sqlbuild_python_front::state_rows::main::migrations::stored_columns_binding::stored_columns_binding; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; const IS_COMPATIBILITY_VIEW: ExecutorHost = ExecutorHost::IsCompatibilityView; use sqlbuild_core::constants::MIGRATION_TABLE_NAME; @@ -219,7 +219,7 @@ fn project_destinations<'py>(project: &Bound<'py, PyAny>) -> PyResult = project.py(); let destinations: Bound<'py, PyDict> = PyDict::new(py); for (key, model) in model_destinations(project)? { - destinations.set_item(key.to_python(py)?, model_reads::name_object(&model)?)?; + destinations.set_item(key.to_python(py)?, name_object(&model)?)?; } Ok(destinations) } @@ -377,7 +377,10 @@ pub(crate) fn pending_migration_origins( if listed.contains(&destination.folded()) { continue; } - let model_name: String = model_reads::model_name(&model)?; + let model_name: String = + sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name( + &model, + )?; for origin in migration_origin_keys(&raw, destination.database.as_deref(), &schemas) { let folded: Folded = Key { database: origin.database, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs index 671e95bf4b..c8b64ad572 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs @@ -28,13 +28,13 @@ use crate::bindings::_helpers::janitor::janitor_keys::{ use crate::bindings::_helpers::janitor::janitor_old_names::{ OldNamePlanning, pending_migration_origins, plan_old_name_views, }; -use crate::bindings::_helpers::relation_naming::fitting::shared_error; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_project::relation_naming::main::shared_error::shared_error; const LIFECYCLE: ExecutorHost = ExecutorHost::OperationLifecycle; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs index dbbccf2edb..0346325648 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs @@ -11,11 +11,12 @@ use pyo3::{pyclass, pymethods}; use sqlbuild_analysis::lineage_graph::main::edge_index::edge_index; use sqlbuild_analysis::lineage_graph::models::{EdgeIndex, EdgeKey}; -use crate::bindings::_helpers::lineage::lineage_facts::{confidence_member, transform_kind_member}; use crate::bindings::_helpers::lineage::lineage_objects::LineageClasses; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_project::lineage::main::confidence_member::confidence_member; +use sqlbuild_python_project::lineage::main::transform_kind_member::transform_kind_member; /// Where a compact model's facts are: given, or projected from its compiled model on first read. pub(crate) enum CompactColumns { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index 1b11fbaa7d..3ba074d40d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -38,13 +38,6 @@ use crate::bindings::_helpers::lineage::lineage_cache::{ }; use crate::bindings::_helpers::lineage::lineage_index::{graph_index, index_views, lineage_keys}; use crate::bindings::_helpers::lineage::project_lineage::build_direct_semantic_uses; -use crate::bindings::_helpers::planner_selection::selectors::resolve_project_selector_tokens; -use crate::bindings::_helpers::planner_selection::selectors::resolve_project_selectors; -use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; -use crate::bindings::_helpers::planner_selection::selectors::{ - match_project_selector, parse_project_selector, -}; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use sqlbuild_python_base::build_progress::main::transient_status::maybe_status::maybe_status; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; @@ -56,6 +49,12 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_project::planner_selection::main::match_project_selector::match_project_selector; +use sqlbuild_python_project::planner_selection::main::parse_project_selector::parse_project_selector; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; +use sqlbuild_python_project::planner_selection::main::resolve_project_selector_tokens::resolve_project_selector_tokens; +use sqlbuild_python_project::planner_selection::main::resolve_project_selectors::resolve_project_selectors; +use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; const UNKNOWN_SELECTOR_ERROR_CODE: &str = "S007"; const RICH_MODE: &str = "rich"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_facts.rs deleted file mode 100644 index 6d640799ea..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_facts.rs +++ /dev/null @@ -1,208 +0,0 @@ -//! Sequence behaviour of the natively hosted `CompactLineageFacts` record. - -use pyo3::IntoPyObjectExt; -use pyo3::exceptions::PyIndexError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PySlice, PyTuple, PyTupleMethods}; -use sqlbuild_python_base::positional_record; - -use sqlbuild_python_base::boundary::main::call_argument::call_argument; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_base::records::types::RecordId; - -/// `ColumnTransformKind` members by native transform code. -const TRANSFORM_KINDS: [&str; 6] = [ - "DIRECT", - "CAST", - "EXPRESSION", - "AGGREGATION", - "STAR", - "CONSTANT", -]; -/// `ColumnLineageConfidence` members by native confidence code. -const CONFIDENCES: [&str; 3] = ["UNKNOWN", "HIGH", "MEDIUM"]; - -/// The `ColumnTransformKind` member of a native transform code. -pub(crate) fn transform_kind_member(py: Python<'_>, code: usize) -> PyResult> { - coded_member(py, (RecordId::ColumnTransformKind, &TRANSFORM_KINDS), code) -} - -/// The `ColumnLineageConfidence` member of a native confidence code. -pub(crate) fn confidence_member(py: Python<'_>, code: usize) -> PyResult> { - coded_member(py, (RecordId::ColumnLineageConfidence, &CONFIDENCES), code) -} - -/// The enum member at `code` of `members`. -fn coded_member( - py: Python<'_>, - (class, members): (RecordId, &[&str]), - code: usize, -) -> PyResult> { - let Some(member) = members.get(code) else { - return Err(PyIndexError::new_err("tuple index out of range")); - }; - Ok(enum_member(py, class, member)?.unbind()) -} - -/// `CompactLineageFacts.resource_name(index)`: the model's name for a native relation index. -fn resource_name<'py>( - facts: &Bound<'py, PyAny>, - index: &Bound<'py, PyAny>, -) -> PyResult> { - let names: Bound<'py, PyAny> = native_field(facts, "resource_names")?; - let name: Bound<'py, PyAny> = mapping_get(&names, index)?; - if !name.is_none() { - return Ok(name); - } - let pooled: Bound<'py, PyAny> = - native_field(facts, "resource_name_indexes")?.call_method1("get", (index, index))?; - native_field(facts, "string_pool")?.get_item(pooled) -} - -/// `CompactLineageFacts.resource_name(index)`. -pub(crate) fn facts_resource_name( - instance: &Bound<'_, PyAny>, - args: &Bound<'_, PyTuple>, - kwargs: Option<&Bound<'_, PyDict>>, -) -> PyResult> { - let index: Bound<'_, PyAny> = call_argument((args, kwargs), "index", 0)?; - Ok(resource_name(instance, &index)?.unbind()) -} - -positional_record! { - extract; - /// One compact lineage row: output name index, transform and confidence codes, sources. - struct CompactColumnRow<'py> { - name_index: Bound<'py, PyAny>, - transform_code: usize, - confidence_code: usize, - sources: Bound<'py, PyAny>, - } -} - -/// One compact row projected to a `CompiledLineageColumnFact`. -fn project_row(facts: &Bound<'_, PyAny>, row: &Bound<'_, PyAny>) -> PyResult> { - let py: Python<'_> = facts.py(); - let pool: Bound<'_, PyAny> = native_field(facts, "string_pool")?; - let CompactColumnRow { - name_index, - transform_code, - confidence_code, - sources, - }: CompactColumnRow<'_> = row.extract()?; - let mut upstream: Vec> = Vec::new(); - for source in sources.try_iter()? { - let source: Bound<'_, PyTuple> = source?.cast_into::()?; - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("resource_type", pool.get_item(source.get_item(0)?)?)?; - keywords.set_item("resource_name", resource_name(facts, &source.get_item(1)?)?)?; - keywords.set_item("column_name", pool.get_item(source.get_item(2)?)?)?; - upstream.push(new_record( - py, - RecordId::CompiledLineageSourceFact, - (), - Some(&keywords), - )?); - } - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("output_column", pool.get_item(name_index)?)?; - keywords.set_item("upstream_columns", PyTuple::new(py, upstream)?)?; - keywords.set_item( - "transform_kind", - coded_member( - py, - (RecordId::ColumnTransformKind, &TRANSFORM_KINDS), - transform_code, - )?, - )?; - keywords.set_item( - "confidence", - coded_member( - py, - (RecordId::ColumnLineageConfidence, &CONFIDENCES), - confidence_code, - )?, - )?; - Ok(new_record(py, RecordId::CompiledLineageColumnFact, (), Some(&keywords))?.unbind()) -} - -/// `CompactLineageFacts.facts`: every row projected once, in order. -pub(crate) fn facts_projected(instance: &Bound<'_, PyAny>) -> PyResult> { - let projected: Vec> = native_field(instance, "rows")? - .try_iter()? - .map(|row| project_row(instance, &row?)) - .collect::>()?; - Ok(PyTuple::new(instance.py(), projected)?.into_any().unbind()) -} - -/// `len(facts)`. -pub(crate) fn facts_len( - instance: &Bound<'_, PyAny>, - _args: &Bound<'_, PyTuple>, - _kwargs: Option<&Bound<'_, PyDict>>, -) -> PyResult> { - native_field(instance, "rows")? - .len()? - .into_py_any(instance.py()) -} - -/// `facts[index]`: one fact, or a tuple of facts for a slice. -pub(crate) fn facts_getitem( - instance: &Bound<'_, PyAny>, - args: &Bound<'_, PyTuple>, - _kwargs: Option<&Bound<'_, PyDict>>, -) -> PyResult> { - let index: Bound<'_, PyAny> = args.get_item(0)?; - let projected: Bound<'_, PyAny> = native_field(instance, "facts")?; - if index.is_instance_of::() { - return Ok(projected.get_item(index)?.unbind()); - } - let length: isize = isize::try_from(projected.len()?)?; - let requested: isize = index.extract()?; - let normalized: isize = if requested >= 0 { - requested - } else { - length + requested - }; - if normalized < 0 || normalized >= length { - return Err(PyIndexError::new_err(index.unbind())); - } - Ok(projected.get_item(normalized)?.unbind()) -} - -/// `iter(facts)`. -pub(crate) fn facts_iter( - instance: &Bound<'_, PyAny>, - _args: &Bound<'_, PyTuple>, - _kwargs: Option<&Bound<'_, PyDict>>, -) -> PyResult> { - Ok(native_field(instance, "facts")? - .try_iter()? - .into_any() - .unbind()) -} - -/// `facts == other`: item-wise equality with any sequence. -pub(crate) fn facts_eq( - instance: &Bound<'_, PyAny>, - args: &Bound<'_, PyTuple>, - _kwargs: Option<&Bound<'_, PyDict>>, -) -> PyResult> { - let py: Python<'_> = instance.py(); - let other: Bound<'_, PyAny> = args.get_item(0)?; - let sequence: Bound<'_, PyAny> = py.import("collections.abc")?.getattr("Sequence")?; - if !other.is_instance(&sequence)? { - return Ok(py.NotImplemented()); - } - let mine: Bound<'_, PyAny> = native_field(instance, "facts")?; - let theirs: Bound<'_, PyTuple> = PyTuple::new( - py, - other - .try_iter()? - .collect::>>>()?, - )?; - mine.eq(theirs)?.into_py_any(py) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs index 5f73ca5eb7..26032298d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs @@ -1,7 +1,6 @@ pub(crate) mod column_lineage_graph; pub(crate) mod lineage_cache; pub(crate) mod lineage_command; -pub(crate) mod lineage_facts; pub(crate) mod lineage_index; pub(crate) mod lineage_objects; pub(crate) mod project_lineage; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs index 605c5b84a1..da32d00298 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -21,8 +21,6 @@ use sqlbuild_analysis::lineage_graph::main::analysis_dialect::analysis_dialect; use sqlbuild_analysis::lineage_graph::models::AnalysisDialect; use sqlbuild_analysis::semantic_validation::models as validation; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::model_reads::star_flags; use crate::bindings::_helpers::lineage::column_lineage_graph::{ CompactColumns, CompactModel, ProjectColumnLineage, }; @@ -31,11 +29,13 @@ use crate::bindings::_helpers::lineage::project_schema::{ fast_schema, inferred_names, rich_schema, }; use crate::bindings::_helpers::project_assembly::syntax_checks::model_placeholders; -use crate::bindings::models::ProjectCatalog; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::analysis::models::ProjectCatalog; +use sqlbuild_python_project::compiled_views::main::compiled_models::star_flags::star_flags; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; const FAST_MODE: &str = "fast"; const RICH_MODE: &str = "rich"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs index 975b6b7511..587a6747cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs @@ -5,8 +5,9 @@ use sqlbuild_analysis::lineage::models::{ LineageResourceType, LineageSchemaResource, RichSchemaResource, }; -use crate::bindings::_helpers::compiled_views::model_reads::{declared_rows, inferred_rows}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::compiled_models::declared_rows::declared_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_rows::inferred_rows; type TypedColumns = Vec<(String, Option)>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 4169796e01..a1d82374fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -29,8 +29,6 @@ use crate::bindings::_helpers::load::load_selection::{ LoadProject, LoaderDefaults, loader_defaults, loader_write_schemas, target_vars, }; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; -use crate::bindings::_helpers::references::source_relations::render_source_relation; use crate::bindings::_helpers::result_output::documents::format_load_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; @@ -54,6 +52,8 @@ use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_use use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; use sqlbuild_python_front::discovery::models::SecretPolicy; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; +use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; const EXECUTION_COMMAND: &str = "sqb load"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 937ce717d6..aa218631f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -20,9 +20,6 @@ use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotRecord}; use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation; use crate::bindings::_helpers::compile_driver::pipeline::compiled_phase; -use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; -use crate::bindings::_helpers::compiled_project::snapshot_execution::NodeKeys; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::location_command::constants::{EXECUTE_ACTION, PUBLISH_ACTION}; use crate::bindings::_helpers::location_command::runtime::run_runtime; use crate::bindings::_helpers::location_command::snapshot::{ @@ -39,6 +36,9 @@ use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::models::NativeProjectView; +use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationSnapshot; +use sqlbuild_python_project::project_snapshot::types::NodeKeys; fn status(py: Python<'_>, message: &str) -> PyResult<()> { write_stderr(py, &format!("{message}\n")) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 7b5c76219a..879f3acf19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -14,21 +14,9 @@ use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; use sqlbuild_discovery::project_validation::main::validate_orchestration_names::validate_orchestration_names; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; -use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; -use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ - compile_declarations, python_declarations, -}; -use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; -use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; -use crate::bindings::_helpers::compiled_project::snapshot_roles::compile_project_roles; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; -use crate::bindings::_helpers::planner_identity::graph::execution_deps; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; -use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; -use crate::bindings::_helpers::planner_scope::values::project_graph; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::orchestration_reads::main::environment_hash::environment_hash; use sqlbuild_python_base::records::main::fields::field::field; @@ -36,6 +24,17 @@ use sqlbuild_python_front::discovery::main::project::named_node_rows::named_node use sqlbuild_python_front::discovery::main::project::project_facts::project_facts; use sqlbuild_python_front::discovery::main::project::validation_error::validation_error; use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; +use sqlbuild_python_project::compiled_views::models::NativeProjectView; +use sqlbuild_python_project::graph::models::ProjectSelection; +use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; +use sqlbuild_python_project::planner_scope::main::build_planner_scope::build_planner_scope; +use sqlbuild_python_project::planner_scope::main::project_graph::project_graph; +use sqlbuild_python_project::project_snapshot::main::compile_declarations::compile_declarations; +use sqlbuild_python_project::project_snapshot::main::compile_project_roles::compile_project_roles; +use sqlbuild_python_project::project_snapshot::main::model_nodes::model_nodes; +use sqlbuild_python_project::project_snapshot::main::python_declarations::python_declarations; +use sqlbuild_python_project::project_snapshot::main::resource_nodes::resource_nodes; +use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationSnapshot; /// The Python node kind whose functions load sources. const LOADER_KIND: &str = "loader"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs index 4090cb4ff6..b95a1a7b94 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs @@ -6,7 +6,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use serde_json::{Value, json}; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::bindings::_helpers::compiled_project::snapshot_boundary::relative_path; use crate::bindings::_helpers::location_command::constants::{ MODEL_NODE_KIND, SNAPSHOT_HIDDEN_SETTING, SOURCE_NODE_KIND, }; @@ -14,6 +13,7 @@ use sqlbuild_python_base::orchestration_reads::main::missing_secrets::missing_se use sqlbuild_python_base::orchestration_reads::main::provider_class_settings::provider_class_settings; use sqlbuild_python_base::orchestration_reads::main::setting_rows::setting_rows; use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_project::project_snapshot::main::relative_path::relative_path; /// How a consumer's code runs inside other snapshot nodes, besides its own node. #[derive(Clone, Copy)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index c1507efb2e..a7203be03f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -27,11 +27,6 @@ use crate::bindings::_helpers::cli::dispatch::{Invocation, command_facts, wareho use crate::bindings::_helpers::command_runtime::warehouses::{ WarehouseInputs, configured_warehouses, }; -use crate::bindings::_helpers::compiled_project::constants::SQL_TEST_KEY_PREFIX; -use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; -use crate::bindings::_helpers::compiled_project::snapshot_execution::{NodeKeys, WorkerBuild}; -use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; -use crate::bindings::_helpers::compiled_views::project_reads::sql_tests; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::command::compile_snapshot; use crate::bindings::_helpers::location_command::constants::{ @@ -52,6 +47,12 @@ use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; +use sqlbuild_python_project::compiled_project::constants::SQL_TEST_KEY_PREFIX; +use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; +use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationSnapshot; +use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationWorker; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; +use sqlbuild_python_project::project_snapshot::types::NodeKeys; /// Each audit and SQL test the snapshot publishes, with the node keys it checks. fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs index fb8434e43a..ec89f1474c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/builder.rs @@ -13,8 +13,6 @@ use sqlbuild_analysis::manifest::models::ManifestInput; use sqlbuild_core::json::errors::JsonEmitError; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::graph::views::native_graph_from_edges; use crate::bindings::_helpers::manifest::rows::{ ProjectRows, plan_input, project_rows, python_hooks, }; @@ -24,6 +22,8 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_project::graph::main::project::native_graph_from_edges::native_graph_from_edges; +use sqlbuild_python_project::graph::models::NativeProjectGraph; const DATETIME_MODULE: &str = "datetime"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index a3c45ce10a..87df77c561 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -1,8 +1,6 @@ //! Permanent Python host calls and compatibility views of native-owned execution data. -pub(crate) mod model_lookup; pub(crate) mod orchestration_sensors; -pub(crate) mod orchestration_triggers; pub(crate) mod registration; pub(crate) mod resource_costs_state; pub(crate) mod warehouse_metadata; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs index 6216853b0b..f05239f1ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -11,7 +11,6 @@ use sqlbuild_orchestration::sensors::models::{ CustomSensorOutcome, SensorCondition, SensorEvaluationRequest, }; -use crate::bindings::_helpers::compiled_project::snapshot_declarations::python_declarations; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; @@ -24,6 +23,7 @@ use sqlbuild_python_front::discovery::main::python_files::provider_by_name::prov use sqlbuild_python_front::discovery::main::python_files::provider_usages::provider_usages; use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; use sqlbuild_python_front::discovery::models::SecretPolicy; +use sqlbuild_python_project::project_snapshot::main::python_declarations::python_declarations; pub(crate) fn evaluate_sensor( py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs deleted file mode 100644 index 5c789c7eaa..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ /dev/null @@ -1,302 +0,0 @@ -//! Authored trigger controls and bounded calendars evaluated only in the location interpreter. - -use std::collections::{BTreeMap, BTreeSet}; -use std::path::Path; - -use chrono::{DateTime, TimeDelta, Timelike, Utc}; -use chrono_tz::Tz; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods}; -use serde_json::{Value, json}; -use sqlbuild_analysis::graph::models::ProjectGraph; -use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; -use sqlbuild_host::entries::models::UserCodeHost; -use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; -use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; -use sqlbuild_orchestration::sensors::models::SensorDefinition; -use sqlbuild_orchestration::snapshots::models::SnapshotInput; - -use crate::bindings::_helpers::graph::python_selection::ProjectSelection; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::main::within::within; -use sqlbuild_python_base::native_host::constants::CALENDAR_WINDOW_PARAMETER; -use sqlbuild_python_base::native_host::constants::SOURCE_NEW_DATA_CONDITION; -use sqlbuild_python_base::native_host::constants::SOURCE_STALE_CONDITION; -use sqlbuild_python_base::native_host::main::host_entry::host_entry as entry; -use sqlbuild_python_base::orchestration_reads::main::callback_code_hash::callback_code_hash; -use sqlbuild_python_base::orchestration_reads::main::declared_at::declared_at; -use sqlbuild_python_base::orchestration_reads::main::orchestration_code_hash::orchestration_code_hash; -use sqlbuild_python_base::orchestration_reads::models::AuthoredSensor; -use sqlbuild_python_front::discovery::main::python_files::discover_provider_classes::discover_provider_classes; -use sqlbuild_python_front::discovery::main::python_files::provider_by_name::provider_by_name; -use sqlbuild_python_front::discovery::main::python_files::provider_usages::provider_usages; -use sqlbuild_python_front::discovery::models::SecretPolicy; - -/// A job's or group's schedule on a named calendar, as its trigger before the calendar's times fill it. -pub(crate) struct CalendarRequest { - pub(crate) calendar: String, - pub(crate) timezone: String, - pub(crate) trigger: TriggerDefinition, -} - -fn horizon(value: &str) -> PyResult { - let invalid = || { - PyValueError::new_err( - "calendar horizon must be a positive fixed duration using d, h, m or s, within datetime range", - ) - }; - let digits = parse_duration(value).ok_or_else(invalid)?; - if [&digits.years, &digits.months] - .iter() - .any(|digits| !digits.trim_start_matches('0').is_empty()) - { - return Err(invalid()); - } - let seconds = [ - (digits.days, 86_400_i64), - (digits.hours, 3_600), - (digits.minutes, 60), - (digits.seconds, 1), - ] - .into_iter() - .try_fold(0_i64, |total, (digits, factor)| { - let digits = digits.parse::().map_err(|_| invalid())?; - let seconds = digits.checked_mul(factor).ok_or_else(invalid)?; - total.checked_add(seconds).ok_or_else(invalid) - })?; - TimeDelta::try_seconds(seconds).ok_or_else(invalid) -} - -fn calendar_times( - (authored, timezone): (&Bound<'_, PyAny>, &str), - session: &Bound<'_, PyAny>, - providers: &Bound<'_, PyDict>, - start: DateTime, -) -> PyResult { - let py = authored.py(); - timezone - .parse::() - .map_err(|_| PyValueError::new_err("calendar timezone must be an IANA timezone"))?; - let duration = horizon(&authored.getattr("horizon")?.extract::()?)?; - let end = start - .checked_add_signed(duration) - .ok_or_else(|| PyValueError::new_err("calendar horizon is out of datetime range"))?; - let datetime = imported(py, "datetime", "datetime")?; - let zone = imported(py, "zoneinfo", "ZoneInfo")?.call1((timezone,))?; - let window = PyDict::new(py); - for (name, instant) in [("start", start), ("end", end)] { - window.set_item( - name, - datetime - .call_method1("fromisoformat", (instant.to_rfc3339(),))? - .call_method1("astimezone", (&zone,))?, - )?; - } - let window = py - .import("sqlbuild.orchestration")? - .getattr("CalendarWindow")? - .call((), Some(&window))?; - let function = authored.getattr("function")?; - let arguments = PyDict::new(py); - arguments.set_item(CALENDAR_WINDOW_PARAMETER, window)?; - for usage in provider_usages(&function, providers)?.try_iter()? { - let usage = usage?; - if usage.getattr("parameter_name")?.extract::()? == CALENDAR_WINDOW_PARAMETER { - continue; - } - arguments.set_item( - usage.getattr("parameter_name")?, - session.call_method1("get", (usage.getattr("provider_name")?,))?, - )?; - } - let mut times: BTreeSet> = BTreeSet::new(); - for (index, time) in function - .call((), Some(&arguments))? - .try_iter()? - .take(MAX_TICKS_PER_BATCH + 1) - .enumerate() - { - if index == MAX_TICKS_PER_BATCH { - return Err(PyValueError::new_err(format!( - "calendar exceeds {MAX_TICKS_PER_BATCH} upcoming times" - ))); - } - let time = time?; - if !time.is_instance(&datetime)? { - return Err(PyValueError::new_err( - "calendar times must be timezone-aware datetimes", - )); - } - let time = - DateTime::parse_from_rfc3339(&time.call_method0("isoformat")?.extract::()?) - .map_err(|_| { - PyValueError::new_err("calendar times must be timezone-aware datetimes") - })? - .with_timezone(&Utc); - if time < start || time >= end { - return Err(PyValueError::new_err( - "calendar time lies outside its half-open evaluation window", - )); - } - times.insert(time); - } - Ok(Schedule::Calendar { - times, - refresh_at: start + duration / 2, - }) -} - -/// Each calendar request's trigger, with each (calendar, timezone) pair evaluated once. -pub(crate) fn compile_calendars( - py: Python<'_>, - project_dir: &Path, - (calendars, requests): (&BTreeMap>, Vec), - mut definitions: Definitions, -) -> PyResult { - if requests.is_empty() { - return Ok(definitions); - } - let providers = discover_provider_classes( - imported(py, "pathlib", "Path")?.call1((project_dir,))?, - SecretPolicy::Defer.name(), - )?; - let by_name = provider_by_name(&providers)?; - let session = call_with( - &entry(py, UserCodeHost::BuildProviderSession)?, - &[ - ("discovered_providers", providers), - ("allow_shared", PyBool::new(py, false).to_owned().into_any()), - ], - )?; - let now = Utc::now(); - let start = now - .with_nanosecond(now.timestamp_subsec_micros() * 1_000) - .ok_or_else(|| PyValueError::new_err("calendar clock is out of range"))?; - within(&session, |_| { - let mut evaluated: BTreeMap<(String, String), Schedule> = BTreeMap::new(); - for request in requests { - let key = (request.calendar.clone(), request.timezone.clone()); - let schedule = match evaluated.get(&key) { - Some(schedule) => schedule.clone(), - None => { - let calendar = calendars.get(&request.calendar).ok_or_else(|| { - PyValueError::new_err(format!( - "calendar '{}' is not declared by a calendar module", - request.calendar - )) - })?; - let schedule = - calendar_times((calendar, &request.timezone), &session, &by_name, start) - .map_err(|error| { - PyValueError::new_err(format!( - "calendar '{}': {error}", - request.calendar - )) - })?; - evaluated.insert(key, schedule.clone()); - schedule - } - }; - let mut trigger = TriggerDefinition { - schedule, - ..request.trigger - }; - trigger.starts_at.get_or_insert(start); - definitions - .triggers - .insert(trigger.job_key.clone(), trigger); - } - Ok(definitions) - }) -} - -pub(crate) fn compile_sensors( - py: Python<'_>, - (project_dir, input, graph): (&Path, &SnapshotInput, (&ProjectGraph, &ProjectSelection)), - authored: Vec, - mut definitions: Definitions, -) -> PyResult { - let sources = if authored.iter().any(|sensor| sensor.function.is_some()) { - orchestration_code_hash(project_dir)? - } else { - String::new() - }; - for sensor in authored { - if !sensor.targets.is_empty() && !sensor.targets.iter().any(|value| value == &input.target) - { - continue; - } - let condition = match (sensor.when, sensor.function) { - (Some(when), None) => { - let mut condition = json!({"kind": when.kind}); - if when.kind == SOURCE_STALE_CONDITION { - condition["after_seconds"] = when.after_seconds; - } else if when.kind != SOURCE_NEW_DATA_CONDITION { - condition["kinds"] = json!(when.kinds); - } - if let Some(job) = when.job { - condition["job_key"] = json!(job); - } - if let Some(select) = when.select { - let selectors: Vec = - select.split_whitespace().map(str::to_owned).collect(); - condition["selection"] = sensor_nodes(graph, &selectors, &when.kind)?; - } - if let Some(selectors) = when.executed_successfully { - condition["executed_successfully"] = - sensor_nodes(graph, &selectors, "executed_successfully")?; - } - condition - } - (None, Some(function)) => json!({"kind": "custom", "function": sensor.name, - "code_hash": callback_code_hash(function.bind(py), &sources)?}), - _ => { - return Err(PyValueError::new_err( - "sensor needs either a built-in condition or a decorated polling callback; callbacks with when are not supported yet", - )); - } - }; - let definition: SensorDefinition = serde_json::from_value(json!({ - "description": sensor.description, - "job_key": sensor.run, - "when": condition, - "minimum_interval_seconds": sensor.minimum_interval_seconds, - "default_status": sensor.default_status, - "targets": sensor.targets, - "declared_at": declared_at(project_dir, sensor.declared_at)?, - })) - .map_err(|error| PyValueError::new_err(format!("sensor {}: {error}", sensor.name)))?; - definitions.sensors.insert(sensor.name, definition); - } - Ok(definitions) -} - -fn sensor_nodes( - (graph, selected): (&ProjectGraph, &ProjectSelection), - selectors: &[String], - field: &str, -) -> PyResult { - if selectors.is_empty() || selectors.iter().any(|value| value.trim().is_empty()) { - return Err(PyValueError::new_err(format!( - "{field} needs nonempty selectors" - ))); - } - let nodes: BTreeSet = selected - .node_keys(graph, selectors, &[]) - .map_err(|error| PyValueError::new_err(format!("{field}: {error}")))?; - if nodes.is_empty() { - return Err(PyValueError::new_err(format!( - "{field} needs nonempty selectors" - ))); - } - if matches!(field, SOURCE_NEW_DATA_CONDITION | SOURCE_STALE_CONDITION) - && nodes.iter().any(|key| !key.starts_with("source:")) - { - return Err(PyValueError::new_err(format!( - "{field} must select only source nodes" - ))); - } - Ok(json!(nodes)) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs index 6aebbf4d28..61e4e26117 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/warehouse_metadata.rs @@ -34,8 +34,6 @@ use sqlbuild_state::state_tables::main::read_history_sql::read_history_sql; use sqlbuild_state::state_tables::models::HistoryRead; use crate::bindings::_helpers::command_runtime::connection::project_connection_config; -use crate::bindings::_helpers::compiled_project::values::plain_value; -use crate::bindings::_helpers::fingerprints::latest_rows::row_fingerprint; use crate::bindings::_helpers::freshness::state_io::row_record; use crate::bindings::_helpers::freshness::support::presumed_utc_datetime; use crate::bindings::_helpers::query_command::project::{ @@ -51,6 +49,8 @@ use sqlbuild_python_base::warehouse_access::main::read_catalog::read_catalog; use sqlbuild_python_base::warehouse_access::main::read_statistics::read_statistics; use sqlbuild_python_front::authored_values::main::sql_value::sql_value; use sqlbuild_python_front::authored_values::models::SqlValueClasses; +use sqlbuild_python_project::compiled_project::main::plain_value::plain_value; +use sqlbuild_python_project::fingerprints::main::row_fingerprint::row_fingerprint; /// One read-only adapter session serves every page of one synchronization. pub(crate) struct HistorySession<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs index 069c096f37..84d46233af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/observations/publication.rs @@ -8,19 +8,20 @@ use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_quer use sqlbuild_analysis::planner_warehouse::models::CursorTagRequest; use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND}; -use crate::bindings::_helpers::compiled_views::{model_reads, project_reads}; use crate::bindings::_helpers::consumption::relation_reads::relation; use crate::bindings::_helpers::observations::source_capture::take_source_capture; -use crate::bindings::_helpers::planner_phases::objects::config_str; use crate::bindings::_helpers::planner_warehouse::constants::RUN_BOUNDED; use crate::bindings::_helpers::planner_warehouse::cursors::{adapter_cursor_sql, bound_task}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::native_host::main::append_observations::append_observations; -use sqlbuild_python_front::native_host::models::ConsumptionConnection; -use sqlbuild_python_front::native_host::models::Location; use sqlbuild_python_front::native_host::models::ObservationAppend; +use sqlbuild_python_front::native_host::models::{ConsumptionConnection, Location}; +use sqlbuild_python_project::compiled_views::main::compiled_models::dep_rows::dep_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::planner_phases::main::config_str::config_str; /// Dependency kind of a source in a model's `dep_rows`. const SOURCE_DEPENDENCY: &str = "source"; @@ -43,9 +44,9 @@ fn cursor_reads( ) -> PyResult> { let succeeded: HashSet<&str> = successful_models.iter().map(String::as_str).collect(); let mut reads: Vec = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model: Bound<'_, PyAny> = model?; - let name: String = model_reads::name_object(&model)?.extract()?; + let name: String = name_object(&model)?.extract()?; if !succeeded.contains(name.as_str()) || config_str(&model, "materialized")?.as_deref() != Some("incremental") { @@ -54,7 +55,7 @@ fn cursor_reads( let Some(cursor_column) = config_str(&model, "cursor")? else { continue; }; - let destination: Bound<'_, PyAny> = model_reads::destination(&model)?; + let destination: Bound<'_, PyAny> = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?; let qualified: Bound<'_, PyAny> = native_field(&destination, "qualified_name")?; if qualified.is_none() || native_field(&destination, "schema")?.is_none() { continue; @@ -90,13 +91,13 @@ fn source_column_rows( if described.is_empty() { return Ok(BTreeMap::new()); } - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model: Bound<'_, PyAny> = model?; - let destination: Bound<'_, PyAny> = model_reads::destination(&model)?; + let destination: Bound<'_, PyAny> = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?; if native_field(&destination, "schema")?.is_none() { continue; } - let sources: Vec = model_reads::dep_rows(&model)? + let sources: Vec = dep_rows(&model)? .into_iter() .filter(|(kind, name)| kind == SOURCE_DEPENDENCY && described.contains_key(name)) .map(|(_, name)| name) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index b08f4227d3..2e676e42da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -11,8 +11,6 @@ use sqlbuild_rules::rule_catalogue::models::RuleListing; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::rules::catalogue::build_rule_catalogue; use crate::bindings::_helpers::rules::guidance::{format_rule, install_rules_skills}; use crate::bindings::_helpers::rules::results::{ @@ -28,6 +26,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; +use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; use sqlbuild_rules::rule_results::main::format_skipped_models::format_skipped_models; const LIST_ACTION: &str = "list"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs index 6d53e00ba8..2b433681a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs @@ -5,11 +5,11 @@ use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_scope::constants::MODEL; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; const INCREMENTAL_MATERIALIZATION: &str = "incremental"; const CREATE_TABLE_ACTION: &str = "create_table"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 149d4844fc..f5a3ad7932 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -3,9 +3,6 @@ use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; @@ -14,6 +11,9 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_front::contracts::main::matching_dynamic_families::matching_dynamic_families; use sqlbuild_python_front::model_config::main::config::config_text::config_text; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::resource_views::main::function_reads::language::language; use sqlbuild_analysis::planner_changes::constants::{ CASE_FOLDING_DIALECT, EMPTY_FINGERPRINT_METADATA_JSON, @@ -31,20 +31,14 @@ use sqlbuild_analysis::planner_changes::models::{ SchemaFindingRow, }; -use crate::bindings::_helpers::compiled_views::model_reads::{inferred_rows, star_flags}; use crate::bindings::_helpers::planner_changes::renames::origin_names; use crate::bindings::_helpers::planner_changes::results::{ backfill_result, change_kind, planner_enum, replay_policy_error, schema_findings, }; -use crate::bindings::_helpers::planner_identity::hashing::{ - baselined_function_definition_hash, baselined_model_query_hash, -}; use crate::bindings::_helpers::planner_identity::metadata::{ compiled_function_fingerprint_sql, function_local_hashes, model_declared_columns_hash, model_version_identity_metadata_json, }; -use crate::bindings::_helpers::planner_scope::retained::scope_data; -use crate::bindings::_helpers::semantic_shapes::shapes::snapshot_validity_columns; use sqlbuild_project::types::InferredRow; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -55,6 +49,13 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::type_system::main::types_equal::types_equal as native_types_equal; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::star_flags::star_flags; +use sqlbuild_python_project::planner_identity::main::baselined_function_definition_hash::baselined_function_definition_hash; +use sqlbuild_python_project::planner_identity::main::baselined_model_query_hash::baselined_model_query_hash; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::resource_views::main::function_reads::key::key; +use sqlbuild_python_project::semantic_shapes::main::snapshot_validity_columns::snapshot_validity_columns; const CHANGES_LOGGER: &str = "sqlbuild.planner.changes"; const REPLAY_CONFIG_KEY: &str = "replay_on_change"; @@ -114,20 +115,21 @@ pub(crate) fn detect_changes<'py>( let query_change_tracking: bool = native_field(&native_field(project, "settings")?, "query_change_tracking")?.is_truthy()?; let dialect: Option = native_field(snapshot, "column_dialect")?.extract()?; - for function in project_reads::functions(project)?.try_iter()? { + for function in + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)? + .try_iter()? + { let function: Bound<'py, PyAny> = function?; - let name: Bound<'py, PyAny> = function_reads::name(&function)?; + let name: Bound<'py, PyAny> = + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)?; let fingerprint_sql: String = compiled_function_fingerprint_sql(&function)?; let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = vec![( "fingerprint_sql", PyString::new(py, &fingerprint_sql).into_any(), )]; - if selected - .selected - .contains(&crate::bindings::_helpers::graph::keys::native_key( - &function_reads::key(&function)?, - )?) - { + if selected.selected.contains( + &sqlbuild_python_project::graph::main::nodes::native_key::native_key(&key(&function)?)?, + ) { let reason: &str = if full_refresh { "full_refresh" } else { @@ -140,9 +142,7 @@ pub(crate) fn detect_changes<'py>( &name.extract::()?, ( &fingerprint_sql, - &function_reads::language(&function)? - .str()? - .extract::()?, + &language(&function)?.str()?.extract::()?, dialect.as_deref(), ), &fingerprint, @@ -227,7 +227,7 @@ pub(crate) fn scope_model_changes<'py>( } }; let function_local_hashes: Bound<'py, PyAny> = function_local_hashes( - &project_reads::functions(project)?, + &sqlbuild_python_project::compiled_views::main::project::functions::functions(project)?, native_field(project, "sql_analysis_dialect")? .extract::>()? .as_deref(), @@ -260,7 +260,7 @@ pub(crate) fn scope_model_changes<'py>( let Some(model) = scope.model(py, &key.1) else { continue; }; - let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; + let name: Bound<'py, PyAny> = name_object(&model)?; let expected: (Option, Option) = ( lookup(&version_hashes, &name)?, lookup(&metadata_jsons, &name)?, @@ -284,7 +284,7 @@ pub(crate) fn model_change<'py>( ) -> PyResult> { let py: Python<'py> = model.py(); let snapshot: &Bound<'py, PyAny> = &pass.snapshot; - let model_name: Bound<'py, PyAny> = model_reads::name_object(model)?; + let model_name: Bound<'py, PyAny> = name_object(model)?; let columns_hash: String = model_declared_columns_hash(model)?; let base_metadata: String = match expected_metadata_json.filter(|text| !text.is_empty()) { Some(text) => text, @@ -322,7 +322,7 @@ pub(crate) fn model_change<'py>( py, &model_name.extract::()?, ( - &model_reads::query_sql(model)?.extract::()?, + &query_sql(model)?.extract::()?, dialect.extract::>()?.as_deref(), ), &fingerprint, @@ -368,7 +368,7 @@ pub(crate) fn model_change<'py>( let renames: Bound<'py, PyAny> = native_field(snapshot, "reference_renames")?; let origins: Option> = match (&compiled_query_hash, &recorded) { (Some(_), Some(recorded)) if definition_changed && renames.is_truthy()? => origin_names( - (model_reads::query_sql(model)?.extract()?, &fingerprint), + (query_sql(model)?.extract()?, &fingerprint), &renames, (&recorded.definition_hash, &dialect), )?, @@ -477,7 +477,7 @@ fn log_query_check( &log_sql, &[ ("logger", logger.clone()), - ("sql", model_reads::query_sql(model)?), + ("sql", query_sql(model)?), ("action", text("compiled_query")), ], )?; @@ -508,7 +508,10 @@ fn model_schema_findings<'py>( for column in &validity { warehouse.retain(|(name, _)| !column.eq_ignore_ascii_case(name)); } - let schema_entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; + let schema_entry: Bound<'py, PyAny> = + sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( + model, + )?; let mut yml: Vec<(String, String)> = Vec::new(); let mut type_enforcement: bool = false; let mut dynamic: Option> = None; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs index 71ae68c2c9..37a0ab4c90 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -1,8 +1,8 @@ //! Model renames that downstream `__ref` names may have been rewritten for. -use crate::bindings::_helpers::fingerprints::query_hashes::compute_query_hash; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_project::fingerprints::main::compute_query_hash::compute_query_hash; use std::collections::BTreeSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -13,9 +13,6 @@ use sqlbuild_analysis::planner_changes::main::origin_reference_names::origin_ref use sqlbuild_analysis::planner_changes::models::{NameSpan, RenameCandidate}; use sqlbuild_rules::lint_text::main::neutralize_interpolation::neutralize_interpolation; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::planner_identity::hashing::baselined_model_query_hash; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::value_error::value_error; @@ -24,6 +21,12 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::compiled_models::references::references; +use sqlbuild_python_project::planner_identity::main::baselined_model_query_hash::baselined_model_query_hash; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; const GENERIC_DIALECT: &str = "generic"; const FINAL_NEWLINE: &str = "\n"; @@ -235,13 +238,12 @@ fn changed_referencing_query( let ref_kind: Bound<'_, PyAny> = enum_member(py, RecordId::SqlReferenceKind, "REF")?; let dialect: Option = native_field(snapshot, "column_dialect")?.extract()?; for model in selected { - let recorded: Bound<'_, PyAny> = - mapping_get(&recorded_models, model_reads::name_object(&model)?)?; + let recorded: Bound<'_, PyAny> = mapping_get(&recorded_models, name_object(&model)?)?; if recorded.is_none() { continue; } let mut references_models: bool = false; - for reference in model_reads::references(&model)?.try_iter()? { + for reference in references(&model)?.try_iter()? { if native_field(&reference?, "ref_kind")?.eq(&ref_kind)? { references_models = true; break; @@ -250,11 +252,8 @@ fn changed_referencing_query( if references_models && baselined_model_query_hash( py, - &model_reads::model_name(&model)?, - ( - &model_reads::query_sql(&model)?.extract::()?, - dialect.as_deref(), - ), + &model_name(&model)?, + (&query_sql(&model)?.extract::()?, dialect.as_deref()), &recorded, )? != native_field(&recorded, "definition_hash")?.extract::()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs index 3ecf29754a..d958342e66 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -1,28 +1,16 @@ //! Clone planning: destination and origin entries, and deferred-clone graph traversal. -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::graph::keys::KeyCache; -use crate::bindings::_helpers::graph::node_set::PyNodeSet; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::graph::views::edge_dict; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_entries::values::{ native, planner_model, planner_type, string, }; -use crate::bindings::_helpers::planner_identity::graph::{execution_deps, execution_order}; use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; use crate::bindings::_helpers::planner_resolution::locations::build_model_locations; use crate::bindings::_helpers::planner_resolution::locations::build_seed_locations; use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest; use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; use crate::bindings::_helpers::planner_resolution::orchestration::{Resolution, resolve_model_sql}; -use crate::bindings::_helpers::planner_selection::selectors::resolve_graph_selection; use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; use crate::bindings::_helpers::planner_warehouse::source_columns::gather_source_columns; use pyo3::PyRef; @@ -41,6 +29,20 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::graph::main::project::edge_dict::edge_dict; +use sqlbuild_python_project::graph::models::KeyCache; +use sqlbuild_python_project::graph::models::NativeProjectGraph; +use sqlbuild_python_project::graph::models::PyNodeSet; +use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; +use sqlbuild_python_project::planner_identity::main::execution_order::execution_order; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_selection::main::resolve_graph_selection::resolve_graph_selection; +use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; +use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; const ENABLED_KEY: &str = "enabled"; @@ -125,7 +127,7 @@ struct CloneResolution<'py> { /// Whether a selected, enabled model is recreated from its SQL rather than copied. fn recreated_view(model: &Bound<'_, PyAny>, selected: &Bound<'_, PyAny>) -> PyResult { - Ok(selected.contains(model_reads::model_key(model)?)? + Ok(selected.contains(model_key(model)?)? && !disabled_model(model)? && materialization_of(model)?.eq(VIEW_MATERIALIZATION)?) } @@ -140,15 +142,20 @@ fn resolved_source_entries<'py>( let source_type = enum_member(py, RecordId::CompiledResourceType, "SOURCE")?; let read = PySet::empty(py)?; let mut readers: Vec> = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; if recreated_view(&model, &selected)? { readers.push(model); } } - for function in project_reads::functions(project)?.try_iter()? { + for function in + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)? + .try_iter()? + { let function = function?; - if selected.contains(function_reads::key(&function)?)? { + if selected.contains( + sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?, + )? { readers.push(function); } } @@ -163,8 +170,10 @@ fn resolved_source_entries<'py>( let mut entries: Vec> = Vec::new(); for source in native_field(source_project, "sources")?.try_iter()? { let source = source?; - if read.contains(source_reads::name(&source)?)? { - entries.push(source_reads::source_entry(&source)?); + if read.contains( + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)?, + )? { + entries.push(source_entry(&source)?); } } tuple_of(py, entries) @@ -176,18 +185,25 @@ fn clone_resolution<'py>( (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { let py = project.py(); - let model_locations = object(py, build_model_locations(&project_reads::models(project)?)?)?; - let seed_locations = object(py, build_seed_locations(&project_reads::seeds(project)?)?)?; + let model_locations = object(py, build_model_locations(&compiled_models(project)?)?)?; + let seed_locations = object( + py, + build_seed_locations( + &sqlbuild_python_project::compiled_views::main::project::seeds::seeds(project)?, + )?, + )?; let function_locations = object( py, - build_function_locations(&project_reads::functions(project)?)?, + build_function_locations( + &sqlbuild_python_project::compiled_views::main::project::functions::functions(project)?, + )?, )?; let source_map = PyDict::new(py); for source in native_field(source_project, "sources")?.try_iter()? { let source = source?; source_map.set_item( - source_reads::name(&source)?, - source_reads::source_entry(&source)?, + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)?, + source_entry(&source)?, )?; } let source_warehouse_columns = object( @@ -223,8 +239,8 @@ fn model_entry<'py>( py, RecordId::ModelPlanEntry, &[ - ("key", model_reads::model_key(model)?), - ("name", model_reads::name_object(model)?), + ("key", model_key(model)?), + ("name", name_object(model)?), ("relative_path", native_field(model, "relative_path")?), ("materialization_type", materialization_type), ("action", planner_type(py, RecordId::PlanAction, action)?), @@ -232,7 +248,7 @@ fn model_entry<'py>( "reason", planner_type(py, RecordId::PlanReason, "NO_CHANGE")?, ), - ("destination", model_reads::destination(model)?), + ("destination", sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(model)?), ("fingerprint_query_sql", string(py, "")), ("resolved_sql", resolved_sql), ("logical_ddl", string(py, "")), @@ -248,17 +264,27 @@ fn clone_model_entries<'py>( let py = project.py(); let selected = native_field(plan, "selected_keys")?; let entries = PyDict::new(py); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; - if !selected.contains(model_reads::model_key(&model)?)? || disabled_model(&model)? { + if !selected.contains(model_key(&model)?)? || disabled_model(&model)? { continue; } let materialization_type = materialization_of(&model)?; let resolved_sql = if materialization_type.eq(VIEW_MATERIALIZATION)? { let functions = PyDict::new(py); - for function in project_reads::functions(project)?.try_iter()? { + for function in + sqlbuild_python_project::compiled_views::main::project::functions::functions( + project, + )? + .try_iter()? + { let function = function?; - functions.set_item(function_reads::name(&function)?, function)?; + functions.set_item( + sqlbuild_python_project::resource_views::main::function_reads::name::name( + &function, + )?, + function, + )?; } let context = Bound::new( py, @@ -303,7 +329,7 @@ fn clone_model_entries<'py>( string(py, "") }; entries.set_item( - model_reads::model_key(&model)?, + model_key(&model)?, model_entry(&model, materialization_type, resolved_sql)?, )?; } @@ -317,11 +343,14 @@ fn clone_source_entries<'py>( ) -> PyResult> { let py = project.py(); let entries = PyDict::new(py); - for source in project_reads::sources(project)?.try_iter()? { + for source in sqlbuild_python_project::compiled_views::main::project::sources::sources(project)? + .try_iter()? + { let source = source?; - let entry = source_reads::source_entry(&source)?; - if !selected.contains(source_reads::key(&source)?)? - || !native_field(&entry, "managed")?.is_truthy()? + let entry = source_entry(&source)?; + if !selected.contains( + sqlbuild_python_project::resource_views::main::source_reads::key::key(&source)?, + )? || !native_field(&entry, "managed")?.is_truthy()? || native_field(&entry, "loader")?.is_none() || !native_field(&entry, "expression")?.is_none() { @@ -352,13 +381,23 @@ fn clone_source_entries<'py>( }), )?; entries.set_item( - source_reads::key(&source)?, + sqlbuild_python_project::resource_views::main::source_reads::key::key(&source)?, planner_model( py, RecordId::CloneSourcePlanEntry, &[ - ("key", source_reads::key(&source)?), - ("name", source_reads::name(&source)?), + ( + "key", + sqlbuild_python_project::resource_views::main::source_reads::key::key( + &source, + )?, + ), + ( + "name", + sqlbuild_python_project::resource_views::main::source_reads::name::name( + &source, + )?, + ), ("destination", destination), ], )?, @@ -375,9 +414,14 @@ fn clone_function_entries<'py>( let py = project.py(); let selected = native_field(plan, "selected_keys")?; let entries = PyDict::new(py); - for function in project_reads::functions(project)?.try_iter()? { + for function in + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)? + .try_iter()? + { let function = function?; - if !selected.contains(function_reads::key(&function)?)? { + if !selected.contains( + sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?, + )? { continue; } let body_sql = resolve_function_sql( @@ -427,7 +471,10 @@ fn clone_function_entries<'py>( ("packages", field("packages")?), ], )?; - entries.set_item(function_reads::key(&function)?, entry)?; + entries.set_item( + sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?, + entry, + )?; } in_order(&entries, &native_field(plan, "execution_order")?) } @@ -438,17 +485,17 @@ fn seed_entry<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { py, RecordId::SeedPlanEntry, &[ - ("key", seed_reads::key(seed)?), - ("name", seed_reads::name(seed)?), - ("destination", seed_reads::destination(seed)?), + ("key", sqlbuild_python_project::resource_views::main::seed_reads::key::key(seed)?), + ("name", sqlbuild_python_project::resource_views::main::seed_reads::name::name(seed)?), + ("destination", sqlbuild_python_project::resource_views::main::seed_reads::destination::destination(seed)?), ( "file_path", - native_field(&seed_reads::seed_file(seed)?, "file_path")?, + native_field(&seed_file(seed)?, "file_path")?, ), ("columns", PyTuple::empty(py).into_any()), ( "csv_settings", - native_field(&seed_reads::schema_entry(seed)?, "csv_settings")?, + native_field(&schema_entry(seed)?, "csv_settings")?, ), ], ) @@ -461,7 +508,9 @@ fn seed_entries<'py>( wanted: &Bound<'py, PyAny>, ) -> PyResult> { let mut entries: Vec> = Vec::new(); - for seed in project_reads::seeds(project)?.try_iter()? { + for seed in + sqlbuild_python_project::compiled_views::main::project::seeds::seeds(project)?.try_iter()? + { let seed = seed?; if wanted.contains(native_field(&seed, field)?)? { entries.push(seed_entry(&seed)?); @@ -476,9 +525,9 @@ fn origin_model_entries<'py>( ) -> PyResult> { let py = project.py(); let mut entries: Vec> = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; - if !selected_names.contains(model_reads::name_object(&model)?)? || disabled_model(&model)? { + if !selected_names.contains(name_object(&model)?)? || disabled_model(&model)? { continue; } let materialization_type = materialization_of(&model)?; @@ -503,12 +552,18 @@ fn origin_source_entries<'py>( let py = project.py(); let mut keys: Vec> = Vec::new(); let mut order: Vec> = Vec::new(); - for source in project_reads::sources(project)?.try_iter()? { + for source in sqlbuild_python_project::compiled_views::main::project::sources::sources(project)? + .try_iter()? + { let source = source?; - if selected_names.contains(source_reads::name(&source)?)? { - keys.push(source_reads::key(&source)?); + if selected_names.contains( + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)?, + )? { + keys.push( + sqlbuild_python_project::resource_views::main::source_reads::key::key(&source)?, + ); } - order.push(source_reads::key(&source)?); + order.push(sqlbuild_python_project::resource_views::main::source_reads::key::key(&source)?); } clone_source_entries( project, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index 9adb537838..8e402988e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -1,10 +1,5 @@ //! Static plans for commands that do not build models, and direct seed identity reads. -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use crate::bindings::_helpers::planner_entries::models::RelationsRequest; use crate::bindings::_helpers::planner_entries::plan_output::{ function_entry, selected_test_entries, test_selected, @@ -15,7 +10,6 @@ use crate::bindings::_helpers::planner_entries::values::{ }; use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; use crate::bindings::_helpers::planner_identity::metadata::build_seed_identity; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; use crate::bindings::_helpers::planner_strategy::audit_entries::build_selected_audit_entries; use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; @@ -33,6 +27,15 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; +use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; +use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::resource_views::main::function_reads::deps::deps; +use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; +use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; use std::collections::{BTreeMap, BTreeSet}; /// Optional fields of a static command plan; absent ones take `PlanOutput` defaults. @@ -162,10 +165,10 @@ pub(crate) fn build_audit_command_plan<'py>( ) -> PyResult> { let py = project.py(); let materializations = PyDict::new(py); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; materializations.set_item( - model_reads::name_object(&model)?, + name_object(&model)?, native_field(&materialization_of(&model)?, "value")?, )?; } @@ -203,9 +206,15 @@ fn called_function_keys<'py>( ) -> PyResult> { let py: Python<'py> = project.py(); let functions: Bound<'py, PyDict> = PyDict::new(py); - for function in project_reads::functions(project)?.try_iter()? { + for function in + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)? + .try_iter()? + { let function = function?; - functions.set_item(function_reads::key(&function)?, function)?; + functions.set_item( + sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?, + function, + )?; } let called: Bound<'py, PySet> = PySet::empty(py)?; let mut pending: Vec> = direct; @@ -217,7 +226,7 @@ fn called_function_keys<'py>( let Some(function) = functions.get_item(&key)? else { continue; }; - for dependency in function_reads::deps(&function)?.try_iter()? { + for dependency in deps(&function)?.try_iter()? { let dependency = dependency?; if functions.contains(&dependency)? { pending.push(dependency); @@ -257,7 +266,7 @@ pub(crate) fn build_test_command_plan<'py>( )?; let (entries, warnings) = (selected.entries, selected.warnings); let mut case_names: BTreeSet = BTreeSet::new(); - for test in project_reads::sql_tests(&project)?.try_iter()? { + for test in sql_tests(&project)?.try_iter()? { let test = test?; let case_name = native_field(&test, "case_name")?; if !case_name.is_none() && test_selected(&test, &selected_keys, &selection)? { @@ -274,9 +283,14 @@ pub(crate) fn build_test_command_plan<'py>( } let function_keys = called_function_keys(&project, function_keys)?; let mut functions: Vec> = Vec::new(); - for function in project_reads::functions(&project)?.try_iter()? { + for function in + sqlbuild_python_project::compiled_views::main::project::functions::functions(&project)? + .try_iter()? + { let function = function?; - if !function_keys.contains(function_reads::key(&function)?)? { + if !function_keys.contains( + sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?, + )? { continue; } let fingerprint_sql = object(py, build_compiled_function_fingerprint_sql(&function)?)?; @@ -334,14 +348,19 @@ pub(crate) fn build_seed_command_plan<'py>( }; let selected_keys = native_field(scope, "selected_keys")?; let mut entries: Vec> = Vec::new(); - for seed in project_reads::seeds(project)?.try_iter()? { + for seed in seeds(project)?.try_iter()? { let seed = seed?; - if !selected_keys.contains(seed_reads::key(&seed)?)? { + if !selected_keys + .contains(sqlbuild_python_project::resource_views::main::seed_reads::key::key(&seed)?)? + { continue; } let identity = object(py, build_seed_identity(&seed)?)?; let (version_hash, metadata_json) = (identity.get_item(0)?, identity.get_item(1)?); - let previous = mapping_get(&existing, seed_reads::name(&seed)?)?; + let previous = mapping_get( + &existing, + sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)?, + )?; let reason = if previous.is_none() { "FIRST_RUN" } else if native_field(&previous, "version_hash")?.ne(&version_hash)? { @@ -353,17 +372,17 @@ pub(crate) fn build_seed_command_plan<'py>( py, RecordId::SeedPlanEntry, &[ - ("key", seed_reads::key(&seed)?), - ("name", seed_reads::name(&seed)?), - ("destination", seed_reads::destination(&seed)?), + ("key", sqlbuild_python_project::resource_views::main::seed_reads::key::key(&seed)?), + ("name", sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)?), + ("destination", sqlbuild_python_project::resource_views::main::seed_reads::destination::destination(&seed)?), ( "file_path", - native_field(&seed_reads::seed_file(&seed)?, "file_path")?, + native_field(&seed_file(&seed)?, "file_path")?, ), ("columns", object(py, extract_seed_columns(&seed)?)?), ( "csv_settings", - native_field(&seed_reads::schema_entry(&seed)?, "csv_settings")?, + native_field(&schema_entry(&seed)?, "csv_settings")?, ), ("fingerprint_definition", metadata_json.clone()), ("fingerprint_version_hash", version_hash), @@ -409,7 +428,10 @@ pub(crate) fn read_selected_seed_fingerprints<'py>( BTreeMap::new(); for seed in seeds.try_iter()? { let seed = seed?; - let destination = seed_reads::destination(&seed)?; + let destination = + sqlbuild_python_project::resource_views::main::seed_reads::destination::destination( + &seed, + )?; let schema = native_field(&destination, "schema")?; if schema.is_none() { continue; @@ -420,7 +442,10 @@ pub(crate) fn read_selected_seed_fingerprints<'py>( .entry((database.clone().unwrap_or_default(), schema.clone())) .or_insert_with(|| ((database, schema), BTreeSet::new())) .1 - .insert(seed_reads::name(&seed)?.extract()?); + .insert( + sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)? + .extract()?, + ); } let fingerprints = PyDict::new(py); let mut relations_by_target: Vec> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs index bed57d5494..0a182d7c85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -1,15 +1,9 @@ //! Display-only plans for external plan surfaces when warehouse planning cannot resolve. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::graph::keys::KeyCache; -use crate::bindings::_helpers::graph::views::edge_dict; use crate::bindings::_helpers::planner_entries::values::{ boolean, native, none, planner_model, planner_type, string, }; -use crate::bindings::_helpers::planner_identity::graph::execution_deps; use crate::bindings::_helpers::planner_phases::full_refresh::resolve_model_full_refresh; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyFrozenSet, PyTuple}; @@ -19,6 +13,14 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::graph::main::project::edge_dict::edge_dict; +use sqlbuild_python_project::graph::models::KeyCache; +use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; +use sqlbuild_python_project::planner_phases::main::object::object; const VIEW: &str = "view"; const INCREMENTAL: &str = "incremental"; @@ -60,7 +62,10 @@ fn display_entry<'py>( resolve_model_full_refresh(model, boolean(py, full_refresh)?.is_truthy()?)?, )? .is_truthy()?; - let query_sql = model_reads::query_sql(model)?; + let query_sql = + sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql( + model, + )?; let custom_name = if materialization_type.eq(CUSTOM)? { config(model, "materialized")? } else { @@ -70,8 +75,8 @@ fn display_entry<'py>( py, RecordId::ModelPlanEntry, &[ - ("key", model_reads::model_key(model)?), - ("name", model_reads::name_object(model)?), + ("key", model_key(model)?), + ("name", name_object(model)?), ("relative_path", native_field(model, "relative_path")?), ( "action", @@ -94,7 +99,7 @@ fn display_entry<'py>( }, )?, ), - ("destination", model_reads::destination(model)?), + ("destination", destination(model)?), ("fingerprint_query_sql", query_sql.clone()), ("resolved_sql", query_sql), ("logical_ddl", string(py, "")), @@ -141,9 +146,9 @@ pub(crate) fn build_display_only_sqlbuild_plan<'py>( .collect::>>()?, )?; let mut entries: Vec> = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; - if selected.contains(model_reads::name_object(&model)?)? { + if selected.contains(name_object(&model)?)? { entries.push(display_entry(&model, full_refresh)?); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index e81fdf2378..1599f248c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -33,7 +33,6 @@ use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; use crate::bindings::_helpers::planner_phases::warehouse_state::gather_planner_warehouse_state; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::source_freshness::build_planner_source_freshness_result; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; @@ -45,6 +44,7 @@ use sqlbuild_python_base::build_command::main::utc_now::utc_now; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::time::Instant; const INSPECTION_CATALOG: AdapterHost = AdapterHost::OpenInspectionCatalog; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index 908a16a460..91bedada89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -1,8 +1,5 @@ //! Per-model plan entries from resolved change results and warehouse snapshot facts. -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; use crate::bindings::_helpers::planner_entries::models::{ @@ -16,7 +13,6 @@ use crate::bindings::_helpers::planner_phases::full_refresh::{ check_full_rebuild_allowed, describe_full_rebuild_cause, resolve_model_full_refresh, }; use crate::bindings::_helpers::planner_phases::models::ModelEntryResults; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::cursor_ranges::{ CursorWindowInputs, config_str, cursor_safety_configs, empty_cursor_inputs, microbatch_range, model_cursor_overrides, plan_cursor_bounds, @@ -33,14 +29,10 @@ use crate::bindings::_helpers::planner_strategy::cursor_inputs::{ use crate::bindings::_helpers::planner_strategy::lineage::reject_capped_watermark_inputs; use crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_configured_microbatch_limits; use crate::bindings::_helpers::planner_strategy::model_entries::assemble_model_plan_entry; -use crate::bindings::_helpers::planner_strategy::models::{ - CursorInputLookups, ModelEntryDecisions, -}; use crate::bindings::_helpers::planner_strategy::strategy::{ materialization_of, resolve_model_plan_action, }; use crate::bindings::_helpers::planner_strategy::values::text; -use crate::bindings::_helpers::semantic_shapes::source_bindings::source_binding_diagnostics; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -51,6 +43,11 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::discovery::main::failures::microbatch_guidance::microbatch_guidance; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::functions::functions; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_strategy::models::{CursorInputLookups, ModelEntryDecisions}; +use sqlbuild_python_project::semantic_shapes::main::source_binding_diagnostics::source_binding_diagnostics; const MICROBATCH_COMPLEXITY_CODE: &str = "MICROBATCH_CONCURRENCY_COMPLEXITY"; @@ -211,11 +208,8 @@ fn plan_model_from_change<'py>( } else { cursor_bounds }; - let warnings = with_cursor_safety_warnings( - &entry_warnings.into_any(), - &model_reads::name_object(model)?, - &bounds, - )?; + let warnings = + with_cursor_safety_warnings(&entry_warnings.into_any(), &name_object(model)?, &bounds)?; Ok((entry, warnings)) } @@ -428,20 +422,22 @@ pub(crate) fn plan_entries<'py>( runtime_cursor_producer_names(&scope, &relations_context.source_map.bind(py).clone())?, )?; let functions_by_name = PyDict::new(py); - for function in project_reads::functions(&project)?.try_iter()? { + for function in functions(&project)?.try_iter()? { let function = function?; - functions_by_name.set_item(function_reads::name(&function)?, function)?; + functions_by_name.set_item( + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)?, + function, + )?; } let settings = native_field(&project, "settings")?; - let scope_data = crate::bindings::_helpers::planner_scope::retained::scope_data(&scope)?; + let scope_data = sqlbuild_python_project::planner_scope::main::scope_data::scope_data(&scope)?; let context = Bound::new( py, ModelPlanContext { model_locations: relations_context.model_locations.clone_ref(py), - models_by_name: - crate::bindings::_helpers::native_host::model_lookup::ModelLookup::Native( - scope_data.clone(), - ), + models_by_name: sqlbuild_python_project::native_host::models::ModelLookup::Native( + scope_data.clone(), + ), functions_by_name: functions_by_name.into_any().unbind(), seed_locations: relations_context.seed_locations.clone_ref(py), function_locations: relations_context.function_locations.clone_ref(py), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs index ac269cd208..e29ce386a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs @@ -1,8 +1,8 @@ //! Typed requests of plan entry construction. -use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use pyo3::prelude::{Bound, Py, PyAny}; use pyo3::pyclass; +use sqlbuild_python_project::native_host::models::ModelLookup; /// What resolving relation locations and source metadata for one scope reads. #[derive(Debug)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs index 6ffe15d500..ee41bcdd48 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -1,23 +1,13 @@ //! `PlanOutput` assembly from planned model entries, seeds, sources, functions, audits and tests. -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps; use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::values::{ error_of, native, none, planner_model, planner_type, string, }; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; -use crate::bindings::_helpers::planner_scope::loaders::upstream_loader_dependency_names; -use crate::bindings::_helpers::planner_scope::sources::build_source_load_entries; use crate::bindings::_helpers::planner_strategy::audit_entries::build_selected_audit_entries; -use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; -use crate::bindings::_helpers::sql_test_glue::models::SqlTestAssembly; -use crate::bindings::_helpers::sql_test_glue::plan_assembly::assemble_sql_test_plans; -use crate::bindings::_helpers::sql_test_glue::plan_driver::sql_test_model_chain_names_by_key; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -28,6 +18,26 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_scope::main::build_source_load_entries::build_source_load_entries; +use sqlbuild_python_project::planner_scope::main::upstream_loader_dependency_names::upstream_loader_dependency_names; +use sqlbuild_python_project::planner_strategy::main::planning_context::planning_context; +use sqlbuild_python_project::resource_views::main::function_reads::arguments::arguments; +use sqlbuild_python_project::resource_views::main::function_reads::entry_point::entry_point; +use sqlbuild_python_project::resource_views::main::function_reads::fingerprint_destination::fingerprint_destination; +use sqlbuild_python_project::resource_views::main::function_reads::language::language; +use sqlbuild_python_project::resource_views::main::function_reads::packages::packages; +use sqlbuild_python_project::resource_views::main::function_reads::relative_path::relative_path; +use sqlbuild_python_project::resource_views::main::function_reads::return_columns::return_columns; +use sqlbuild_python_project::resource_views::main::function_reads::returns::returns; +use sqlbuild_python_project::resource_views::main::function_reads::runtime_version::runtime_version; +use sqlbuild_python_project::resource_views::main::function_reads::source_file_path::source_file_path; +use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; +use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; +use sqlbuild_python_project::sql_test_glue::main::assemble_sql_test_plans::assemble_sql_test_plans; +use sqlbuild_python_project::sql_test_glue::main::sql_test_model_chain_names_by_key_function::sql_test_model_chain_names_by_key_function; +use sqlbuild_python_project::sql_test_glue::models::SqlTestAssembly; const LOADER_CONSUMER: &str = "loader"; const HOOK_CONSUMER: &str = "hook"; @@ -108,27 +118,31 @@ fn seed_entries<'py>( let metadata_jsons = native_field(extras, "seed_metadata_jsons")?; let plan_reasons = native_field(extras, "seed_plan_reasons")?; let mut entries: Vec> = Vec::new(); - for seed in project_reads::seeds(project)?.try_iter()? { + for seed in + sqlbuild_python_project::compiled_views::main::project::seeds::seeds(project)?.try_iter()? + { let seed = seed?; - if !selected_keys.contains(seed_reads::key(&seed)?)? { + if !selected_keys + .contains(sqlbuild_python_project::resource_views::main::seed_reads::key::key(&seed)?)? + { continue; } - let name = seed_reads::name(&seed)?; + let name = sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)?; entries.push(planner_model( py, RecordId::SeedPlanEntry, &[ - ("key", seed_reads::key(&seed)?), + ("key", sqlbuild_python_project::resource_views::main::seed_reads::key::key(&seed)?), ("name", name.clone()), - ("destination", seed_reads::destination(&seed)?), + ("destination", sqlbuild_python_project::resource_views::main::seed_reads::destination::destination(&seed)?), ( "file_path", - native_field(&seed_reads::seed_file(&seed)?, "file_path")?, + native_field(&seed_file(&seed)?, "file_path")?, ), ("columns", object(py, extract_seed_columns(&seed)?)?), ( "csv_settings", - native_field(&seed_reads::schema_entry(&seed)?, "csv_settings")?, + native_field(&schema_entry(&seed)?, "csv_settings")?, ), ( "fingerprint_definition", @@ -376,30 +390,30 @@ pub(super) fn function_entry<'py>( py, RecordId::FunctionPlanEntry, &[ - ("key", function_reads::key(function)?), - ("name", function_reads::name(function)?), - ("relative_path", function_reads::relative_path(function)?), - ("destination", function_reads::destination(function)?), - ("arguments", function_reads::arguments(function)?), - ("returns", function_reads::returns(function)?), + ("key", sqlbuild_python_project::resource_views::main::function_reads::key::key(function)?), + ("name", sqlbuild_python_project::resource_views::main::function_reads::name::name(function)?), + ("relative_path", relative_path(function)?), + ("destination", sqlbuild_python_project::resource_views::main::function_reads::destination::destination(function)?), + ("arguments", arguments(function)?), + ("returns", returns(function)?), ("body_sql", body_sql), ("fingerprint_query_sql", fingerprint_query_sql), ( "fingerprint_destination", - function_reads::fingerprint_destination(function)?, + fingerprint_destination(function)?, ), - ("return_columns", function_reads::return_columns(function)?), - ("language", function_reads::language(function)?), + ("return_columns", return_columns(function)?), + ("language", language(function)?), ( "source_file_path", - function_reads::source_file_path(function)?, + source_file_path(function)?, ), ( "runtime_version", - function_reads::runtime_version(function)?, + runtime_version(function)?, ), - ("entry_point", function_reads::entry_point(function)?), - ("packages", function_reads::packages(function)?), + ("entry_point", entry_point(function)?), + ("packages", packages(function)?), ("previous_query_sql", previous_query_sql), ("reason", reason), ], @@ -416,12 +430,18 @@ fn function_entries<'py>( let functions = native_field(changes, "functions")?; let previous = native_field(&native_field(snapshot, "fingerprints")?, "functions")?; let mut entries: Vec> = Vec::new(); - for function in project_reads::functions(project)?.try_iter()? { + for function in + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)? + .try_iter()? + { let function = function?; - if !selected_keys.contains(function_reads::key(&function)?)? { + if !selected_keys.contains( + sqlbuild_python_project::resource_views::main::function_reads::key::key(&function)?, + )? { continue; } - let name = function_reads::name(&function)?; + let name = + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)?; let change = functions.get_item(&name)?; let previous_query_sql = if previous.contains(&name)? { native_field(&previous.get_item(&name)?, "definition")? @@ -476,7 +496,7 @@ pub(super) fn selected_test_entries<'py>( ) -> PyResult> { let py = project.py(); let mut selected: Vec> = Vec::new(); - for test in project_reads::sql_tests(project)?.try_iter()? { + for test in sql_tests(project)?.try_iter()? { let test = test?; if !test_selected(&test, selected_keys, selection)? { continue; @@ -778,6 +798,6 @@ fn build_selected_test_entries<'py>( } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(sql_test_model_chain_names_by_key, module)?)?; + module.add_function(sql_test_model_chain_names_by_key_function(module)?)?; module.add_function(wrap_pyfunction!(build_selected_test_entries, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs index 10a6299a67..efaf15b9fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs @@ -1,20 +1,13 @@ //! Relation locations and source metadata for plan entry construction. -use crate::bindings::_helpers::compiled_views::audit_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::graph::keys::native_key; use crate::bindings::_helpers::planner_entries::models::{ PlannerRelationsContext, RelationsRequest, }; use crate::bindings::_helpers::planner_entries::values::planner_model; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; use crate::bindings::_helpers::planner_resolution::locations::build_model_locations; use crate::bindings::_helpers::planner_resolution::locations::build_seed_locations; use crate::bindings::_helpers::planner_resolution::refs::apply_deferred_locations; -use crate::bindings::_helpers::planner_scope::retained::scope_data; -use crate::bindings::_helpers::planner_scope::sources::source_load_map; -use crate::bindings::_helpers::planner_scope::values::keys; use crate::bindings::_helpers::planner_strategy::audit_entries::audit_is_selected; use crate::bindings::_helpers::planner_warehouse::semantic_sources::semantic_source_columns; use crate::bindings::_helpers::planner_warehouse::source_columns::gather_source_inspection; @@ -29,6 +22,16 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::project::audits::audits; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::functions::functions; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_scope::main::keys::keys; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_scope::main::source_load_map::source_load_map; +use sqlbuild_python_project::resource_views::main::audit_reads::scope_deps::scope_deps; use std::collections::HashSet; /// Relation keys needed to render selected audits without selecting those relations. @@ -37,10 +40,10 @@ fn selected_audit_relation_keys( selected_keys: &HashSet, ) -> PyResult> { let mut keys: HashSet = HashSet::new(); - for audit in project_reads::audits(project)?.try_iter()? { + for audit in audits(project)?.try_iter()? { let audit = audit?; if audit_is_selected(&audit, selected_keys)? { - for dependency in audit_reads::scope_deps(&audit)?.try_iter()? { + for dependency in scope_deps(&audit)?.try_iter()? { keys.insert(native_key(&dependency?)?); } } @@ -103,15 +106,9 @@ pub(crate) fn planner_relations_context<'py>( deferral }; let selected_keys = native_field(&scope, "selected_keys")?; - let mut model_locations = object( - py, - build_model_locations(&project_reads::models(&project)?)?, - )?; - let mut seed_locations = object(py, build_seed_locations(&project_reads::seeds(&project)?)?)?; - let function_locations = object( - py, - build_function_locations(&project_reads::functions(&project)?)?, - )?; + let mut model_locations = object(py, build_model_locations(&compiled_models(&project)?)?)?; + let mut seed_locations = object(py, build_seed_locations(&seeds(&project)?)?)?; + let function_locations = object(py, build_function_locations(&functions(&project)?)?)?; let deferred_locations = native_field(&deferral, "deferred_locations")?; if !deferred_locations.is_none() { let applied = object( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index b0dddacebb..13e30cf38e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -15,14 +15,9 @@ use sqlbuild_analysis::planner_identity::models::{GraphKey, IdentityGraph, Ident use sqlbuild_core::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::planner_identity::hashing::{baselined_model_query_hash, graph_key}; use crate::bindings::_helpers::planner_identity::metadata::{ function_local_hashes, model_version_identity_metadata_json, seed_identity, }; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -31,6 +26,12 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::planner_identity::main::baselined_model_query_hash::baselined_model_query_hash; +use sqlbuild_python_project::planner_identity::main::graph_key::graph_key; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; fn key_of(kind: &str, name: &str) -> GraphKey { GraphKey { @@ -121,14 +122,24 @@ pub(crate) fn direct_model_version_identities<'py>( let mut graph = IdentityGraph::default(); for function in functions.try_iter()? { let function: Bound<'py, PyAny> = function?; - let name: String = function_reads::name(&function)?.extract()?; + let name: String = + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)? + .extract()?; let Some(function_hash) = function_hashes.get_item(&name)? else { continue; }; graph.nodes.insert( - graph_key(&function_reads::key(&function)?)?, + graph_key( + &sqlbuild_python_project::resource_views::main::function_reads::key::key( + &function, + )?, + )?, IdentityNode { - upstream_keys: graph_keys(&function_reads::deps(&function)?)?, + upstream_keys: graph_keys( + &sqlbuild_python_project::resource_views::main::function_reads::deps::deps( + &function, + )?, + )?, local_hash: Some(function_hash.extract()?), }, ); @@ -139,7 +150,8 @@ pub(crate) fn direct_model_version_identities<'py>( for seed in seeds.try_iter()? { let seed: Bound<'py, PyAny> = seed?; let (seed_hash, metadata_json): (String, String) = seed_identity(&seed)?; - let name: Bound<'py, PyAny> = seed_reads::name(&seed)?; + let name: Bound<'py, PyAny> = + sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)?; seed_hashes.set_item(&name, &seed_hash)?; seed_metadata.set_item(&name, metadata_json)?; } @@ -176,7 +188,7 @@ pub(crate) fn direct_model_version_identities<'py>( let Some(model) = scope.model(py, &order_key.name) else { continue; }; - let name: String = model_reads::model_name(&model)?; + let name: String = model_name(&model)?; let metadata_json: String = model_version_identity_metadata_json( &model, Some(function_hashes.as_any()), @@ -186,10 +198,7 @@ pub(crate) fn direct_model_version_identities<'py>( let query_fingerprint: String = baselined_model_query_hash( py, &name, - ( - &model_reads::query_sql(&model)?.extract::()?, - dialect.as_deref(), - ), + (&query_sql(&model)?.extract::()?, dialect.as_deref()), &recorded_model, )?; let local_hash: String = model_local_identity_hash(&query_fingerprint, &metadata_json); @@ -198,7 +207,11 @@ pub(crate) fn direct_model_version_identities<'py>( graph.nodes.insert( key_of(MODEL_RESOURCE, &name), IdentityNode { - upstream_keys: graph_keys(&model_reads::deps(&model)?)?, + upstream_keys: graph_keys( + &sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps( + &model, + )?, + )?, local_hash: Some(local_hash), }, ); @@ -296,11 +309,15 @@ pub(crate) fn with_honest_model_write_hashes<'py>( let local_hashes: Bound<'py, PyAny> = native_field(&identities, "model_local_hashes")?; for model in scope.models.values() { let model = model.bind(py); - let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; + let name: Bound<'py, PyAny> = name_object(&model)?; graph.nodes.insert( key_of(MODEL_RESOURCE, &name.extract::()?), IdentityNode { - upstream_keys: graph_keys(&model_reads::deps(&model)?)?, + upstream_keys: graph_keys( + &sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps( + &model, + )?, + )?, local_hash: mapping_get(&local_hashes, &name)?.extract()?, }, ); @@ -339,7 +356,7 @@ pub(crate) fn with_honest_model_write_hashes<'py>( let Some(model) = scope.model(py, &order_key.name) else { continue; }; - let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; + let name: Bound<'py, PyAny> = name_object(&model)?; if mapping_get(&local_hashes, &name)?.is_none() { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs index 9508c1d437..f9a3ca7624 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -16,13 +16,7 @@ use sqlbuild_analysis::planner_identity::main::function_fingerprint_sql::functio use sqlbuild_analysis::planner_identity::models::FunctionFingerprintFields; use sqlbuild_core::fingerprint_identity::main::stable_identity_hash::stable_identity_hash; -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; use crate::bindings::_helpers::microbatches::conversion::timestamp; -use crate::bindings::_helpers::planner_identity::hashing::{ - baselined_function_definition_hash, function_definition_hash, -}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; @@ -35,6 +29,20 @@ use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_front::discovery::main::headers::hook_entries::hook_entries; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::config::config; +use sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::planner_identity::main::baselined_function_definition_hash::baselined_function_definition_hash; +use sqlbuild_python_project::planner_identity::main::function_definition_hash::function_definition_hash; +use sqlbuild_python_project::resource_views::main::function_reads::arguments::arguments; +use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; +use sqlbuild_python_project::resource_views::main::function_reads::entry_point::entry_point; +use sqlbuild_python_project::resource_views::main::function_reads::packages::packages; +use sqlbuild_python_project::resource_views::main::function_reads::return_columns::return_columns; +use sqlbuild_python_project::resource_views::main::function_reads::returns::returns; +use sqlbuild_python_project::resource_views::main::function_reads::runtime_version::runtime_version; +use sqlbuild_python_project::resource_views::main::seed_reads::external::external; +use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; const LOCAL_FUNCTION_HASHES_KEY: &str = "local_function_hashes"; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -102,7 +110,7 @@ pub(crate) fn model_version_identity_metadata_json<'py>( let py: Python<'py> = model.py(); let local_function_hashes: Bound<'py, PyDict> = PyDict::new(py); if let Some(function_hashes) = function_local_hashes.filter(|hashes| !hashes.is_none()) { - for upstream_key in model_reads::deps(model)?.try_iter()? { + for upstream_key in deps(model)?.try_iter()? { let upstream_key: Bound<'py, PyAny> = upstream_key?; let kind: String = native_field(&upstream_key, "resource_type")? .str()? @@ -117,11 +125,11 @@ pub(crate) fn model_version_identity_metadata_json<'py>( } } } - let config_values: Bound<'py, PyAny> = native_field(&model_reads::config(model)?, "values")?; + let config_values: Bound<'py, PyAny> = native_field(&config(model)?, "values")?; let signature: Bound<'py, PyDict> = execution_signature(model, &config_values, hook_version_hashes)?; version_identity_metadata_json( - &model_reads::name_object(model)?, + &name_object(model)?, &config_values, ( Some(local_function_hashes.into_any()), @@ -182,7 +190,10 @@ fn contract_output_signature<'py>( config_values: &Bound<'py, PyAny>, ) -> PyResult>> { let py: Python<'py> = model.py(); - let schema_entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; + let schema_entry: Bound<'py, PyAny> = + sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( + model, + )?; if schema_entry.is_none() { return Ok(None); } @@ -196,7 +207,10 @@ fn contract_output_signature<'py>( if !enforced && native_field(&schema_entry, "model_schema")?.is_none() { return Ok(None); } - let enum_columns: Bound<'py, PyAny> = model_reads::enum_columns(model)?; + let enum_columns: Bound<'py, PyAny> = + sqlbuild_python_project::compiled_views::main::compiled_models::enum_columns::enum_columns( + model, + )?; let rendered: Bound<'py, PyList> = PyList::empty(py); for column in columns.try_iter()? { let column: Bound<'py, PyAny> = column?; @@ -259,7 +273,10 @@ fn family_signatures<'py>(families: &Bound<'py, PyAny>) -> PyResult) -> PyResult { let py: Python<'_> = model.py(); - let schema_entry: Bound<'_, PyAny> = model_reads::schema_entry(model)?; + let schema_entry: Bound<'_, PyAny> = + sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( + model, + )?; let payload: Bound<'_, PyDict> = PyDict::new(py); payload.set_item( "contract", @@ -299,7 +316,12 @@ fn recorded_function_unchanged( return Ok(false); } let py: Python<'_> = function.py(); - let language: String = function_reads::language(function)?.str()?.extract()?; + let language: String = + sqlbuild_python_project::resource_views::main::function_reads::language::language( + function, + )? + .str()? + .extract()?; let fingerprint_sql: String = compiled_function_fingerprint_sql(function)?; let current: String = function_definition_hash(py, name, (&fingerprint_sql, &language, dialect))?; @@ -352,36 +374,36 @@ pub(crate) fn function_local_hashes<'py>( let mut recorded_local: Option> = None; for function in functions.try_iter()? { let function: Bound<'py, PyAny> = function?; - let name: String = function_reads::name(&function)?.extract()?; - let language: Bound<'py, PyAny> = - native_field(&function_reads::language(&function)?, "value")?; + let name: String = + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)? + .extract()?; + let language: Bound<'py, PyAny> = native_field( + &sqlbuild_python_project::resource_views::main::function_reads::language::language( + &function, + )?, + "value", + )?; let body_hash: String = function_definition_hash( py, &name, ( - &function_reads::body_sql(&function)?.extract::()?, + &body_sql(&function)?.extract::()?, &language.extract::()?, dialect, ), )?; let payload: Bound<'py, PyDict> = PyDict::new(py); - payload.set_item( - "arguments", - name_type_pairs(&function_reads::arguments(&function)?)?, - )?; - payload.set_item("returns", function_reads::returns(&function)?)?; + payload.set_item("arguments", name_type_pairs(&arguments(&function)?)?)?; + payload.set_item("returns", returns(&function)?)?; payload.set_item( "return_columns", - name_type_pairs(&function_reads::return_columns(&function)?)?, + name_type_pairs(&return_columns(&function)?)?, )?; payload.set_item("body_sql", body_hash)?; payload.set_item("language", language)?; - payload.set_item( - "runtime_version", - function_reads::runtime_version(&function)?, - )?; - payload.set_item("entry_point", function_reads::entry_point(&function)?)?; - payload.set_item("packages", function_reads::packages(&function)?)?; + payload.set_item("runtime_version", runtime_version(&function)?)?; + payload.set_item("entry_point", entry_point(&function)?)?; + payload.set_item("packages", packages(&function)?)?; let mut local_hash: String = stable_identity_hash(&sorted_dumps(payload.as_any(), SPACED, true)?); if recorded_function_unchanged(&function, (&name, dialect), recorded)? { @@ -441,39 +463,50 @@ fn rendered_pairs(items: &Bound<'_, PyAny>) -> PyResult> { /// Stable function definition text used for change fingerprints. pub(crate) fn compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { Ok(function_fingerprint_sql(&FunctionFingerprintFields { - language: function_reads::language(function)?.str()?.extract()?, - arguments: rendered_pairs(&function_reads::arguments(function)?)?, - returns: function_reads::returns(function)?.str()?.extract()?, - return_columns: rendered_pairs(&function_reads::return_columns(function)?)?, - runtime_version: text_of(function_reads::runtime_version(function))?, - entry_point: text_of(function_reads::entry_point(function))?, - packages: function_reads::packages(function)? + language: + sqlbuild_python_project::resource_views::main::function_reads::language::language( + function, + )? + .str()? + .extract()?, + arguments: rendered_pairs(&arguments(function)?)?, + returns: returns(function)?.str()?.extract()?, + return_columns: rendered_pairs(&return_columns(function)?)?, + runtime_version: text_of(runtime_version(function))?, + entry_point: text_of(entry_point(function))?, + packages: packages(function)? .try_iter()? .map(|package| package?.extract::()) .collect::>()?, - body_sql: function_reads::body_sql(function)?.extract()?, + body_sql: body_sql(function)?.extract()?, })) } /// `(identity_hash, metadata_json)` of one compiled seed from its content and load-affecting config. pub(crate) fn seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { let py: Python<'_> = seed.py(); - let schema_entry: Bound<'_, PyAny> = seed_reads::schema_entry(seed)?; + let schema_entry: Bound<'_, PyAny> = + sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry( + seed, + )?; let csv_settings: Bound<'_, PyAny> = native_field(&schema_entry, "csv_settings")?; let columns: Bound<'_, PyList> = PyList::empty(py); for column in native_field(&schema_entry, "columns")?.try_iter()? { columns.append(column_signature(&column?)?)?; } - let rows: Bound<'_, PyAny> = if seed_reads::external(seed)?.is_truthy()? { + let rows: Bound<'_, PyAny> = if external(seed)?.is_truthy()? { PyList::empty(py).into_any() } else { seed_rows( - &native_field(&seed_reads::seed_file(seed)?, "file_path")?, + &native_field(&seed_file(seed)?, "file_path")?, &csv_settings, )? }; let payload: Bound<'_, PyDict> = PyDict::new(py); - payload.set_item("name", seed_reads::name(seed)?)?; + payload.set_item( + "name", + sqlbuild_python_project::resource_views::main::seed_reads::name::name(seed)?, + )?; payload.set_item( "csv_settings", imported(py, DATACLASSES_MODULE, "asdict")?.call1((&csv_settings,))?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs index 2b7ae821ed..f8f6364a35 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs @@ -1,5 +1,3 @@ -pub(crate) mod graph; -pub(crate) mod hashing; pub(crate) mod identities; pub(crate) mod metadata; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs index f9df4e384b..cce0b3f494 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs @@ -2,13 +2,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::planner_identity::{ - graph, hashing, identities, metadata, staleness, -}; +use crate::bindings::_helpers::planner_identity::{identities, metadata, staleness}; +use sqlbuild_python_project::planner_identity::main::register_graph::register_graph; +use sqlbuild_python_project::planner_identity::main::register_hashing::register_hashing; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - graph::register(module)?; - hashing::register(module)?; + register_graph(module)?; + register_hashing(module)?; identities::register(module)?; metadata::register(module)?; staleness::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs index e2b5244dc2..bbdd4ec732 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs @@ -18,13 +18,13 @@ use sqlbuild_analysis::planner_identity::models::{ }; use crate::bindings::_helpers::planner_changes::detect::{ModelPass, model_change}; -use crate::bindings::_helpers::planner_identity::hashing::graph_key; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_identity::main::graph_key::graph_key; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; /// Change kinds that mean the model's own identity changed. const OWN_CHANGE_KINDS: [&str; 5] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs index ee87e56253..6f8df1dd2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs @@ -1,6 +1,5 @@ //! Gather live candidate evidence without changing adapter inspection ownership. -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::{ config, definition_hash, lower, }; @@ -8,11 +7,14 @@ use crate::bindings::_helpers::planner_migrations::models::{ ColumnCandidate, ColumnMigrationInputs, MigrationPhase, }; use crate::bindings::_helpers::planner_migrations::projections::parse_query_shape; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; pub(super) fn candidates<'py>( (phase, columns): (&MigrationPhase<'py>, &ColumnMigrationInputs<'py>), @@ -36,7 +38,7 @@ pub(super) fn candidates<'py>( if !materialized.eq("incremental")? && !materialized.eq("snapshot")? { continue; } - let name = model_reads::name_object(&model)?; + let name = name_object(&model)?; if columns.full_refresh_model_names.contains(&name)? || !native_field(&snapshot, "existing_relations")?.contains(&name)? { @@ -45,7 +47,7 @@ pub(super) fn candidates<'py>( let warehouse = native_field(&snapshot, "existing_columns")? .call_method1("get", (&name, PyTuple::empty(py)))?; let declared = PyDict::new(py); - let schema = model_reads::schema_entry(&model)?; + let schema = schema_entry(&model)?; if !schema.is_none() { for column in native_field(&schema, "columns")?.try_iter()? { let column = column?; @@ -82,7 +84,7 @@ pub(super) fn candidates<'py>( }; let current = parse_query_shape( py, - &model_reads::query_sql(&model)?.extract::()?, + &query_sql(&model)?.extract::()?, dialect.extract::>()?.as_deref(), )?; result.push(ColumnCandidate { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs index 6e6385e0df..1ea1e3457b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -2,15 +2,12 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::model_reads::inferred_rows; use crate::bindings::_helpers::planner_migrations::column_candidates::declared_names; use crate::bindings::_helpers::planner_migrations::column_objects::{lower, model, replace}; use crate::bindings::_helpers::planner_migrations::columns::identical_renames; use crate::bindings::_helpers::planner_migrations::columns::rename_hints; use crate::bindings::_helpers::planner_migrations::constants::{AUTOMATIC, DONE}; use crate::bindings::_helpers::planner_migrations::models::{CheckedColumns, ColumnCandidate}; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; @@ -23,6 +20,9 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::state_rows::main::migrations::newest_column::newest_column; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::planner_phases::main::object::object; pub(super) fn decide<'py>( runtime: &Bound<'py, PyAny>, @@ -69,8 +69,11 @@ pub(super) fn decide<'py>( outputs.push(name.to_lowercase()); } } - let relation = host_entry(py, ExecutorHost::MigrationRelationForLocation)? - .call1((model_reads::destination(&candidate.model)?,))?; + let relation = host_entry(py, ExecutorHost::MigrationRelationForLocation)?.call1(( + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( + &candidate.model, + )?, + ))?; let mut entries: Vec> = Vec::new(); for (origin_name, destination, discovery) in pairs { let origin_key = lower(&origin_name)?; @@ -107,8 +110,8 @@ pub(super) fn decide<'py>( py, RecordId::ColumnMigrationPlanEntry, &[ - ("model_name", model_reads::name_object(&candidate.model)?), - ("destination", model_reads::destination(&candidate.model)?), + ("model_name", name_object(&candidate.model)?), + ("destination", sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&candidate.model)?), ( "origin_column", if let Some(origin) = origin { @@ -162,7 +165,10 @@ pub(super) fn checked<'py>( warnings: Vec::new(), }); } - let destination = model_reads::destination(&candidate.model)?; + let destination = + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( + &candidate.model, + )?; let qualified = native_field(&destination, "qualified_name")?; let relation = if physical.is_truthy()? { physical.clone() @@ -243,7 +249,7 @@ pub(super) fn hints<'py>( Some(keywords( py, &[ - ("model_name", model_reads::name_object(&candidate.model)?), + ("model_name", name_object(&candidate.model)?), ("previous", candidate.previous.clone()), ("current", candidate.current.clone()), ("excluded", excluded.call_method1("union", (claimed,))?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs index 5c552ee897..4c39e4bb25 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs @@ -1,8 +1,5 @@ //! Boundary conversions for the column migration planner. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::planner_identity::hashing::model_definition_hash; -use crate::bindings::_helpers::planner_phases::objects::object; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString}; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -11,6 +8,10 @@ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_key use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::planner_identity::main::model_definition_hash::model_definition_hash; +use sqlbuild_python_project::planner_phases::main::object::object; pub(super) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() @@ -61,8 +62,8 @@ pub(super) fn definition_hash<'py>( model.py(), model_definition_hash( model.py(), - &model_reads::name_object(model)?.extract::()?, - &model_reads::query_sql(model)?.extract::()?, + &name_object(model)?.extract::()?, + &query_sql(model)?.extract::()?, native_field(snapshot, "column_dialect")? .extract::>()? .as_deref(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs index 4a89487ac9..f8302572dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs @@ -1,7 +1,5 @@ //! Project column renames onto the existing snapshot and redirect cursor reads. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use crate::bindings::_helpers::planner_migrations::column_objects::{ config, definition_hash, lower, native, replace, }; @@ -10,8 +8,6 @@ use crate::bindings::_helpers::planner_migrations::models::{ ColumnMigrationInputs, MigrationPhase, }; use crate::bindings::_helpers::planner_migrations::projections::renames_explain_change; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::boundary::main::imported::imported; @@ -19,6 +15,12 @@ use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::native_host::models::ModelLookup; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; pub(super) fn overlay<'py>( snapshot: &Bound<'py, PyAny>, @@ -36,7 +38,7 @@ pub(super) fn overlay<'py>( let fingerprints = imported(py, "builtins", "dict")?.call1((native_field(&stored, "models")?,))?; for candidate in candidates { - let name = model_reads::name_object(&candidate.model)?; + let name = name_object(&candidate.model)?; let renames = PyDict::new(py); let pending = PyDict::new(py); let mut blocked = false; @@ -96,7 +98,7 @@ pub(super) fn overlay<'py>( replace( &candidate.fingerprint, &[ - ("definition", model_reads::query_sql(&candidate.model)?), + ("definition", query_sql(&candidate.model)?), ( "definition_hash", definition_hash(&candidate.model, snapshot)?, @@ -160,7 +162,7 @@ pub(super) fn cursors<'py>( physical } else { let qualified = native_field( - &model_reads::destination(&models.get(py, &name.extract::()?)?)?, + &destination(&models.get(py, &name.extract::()?)?)?, "qualified_name", )?; if qualified.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 2f7c689c54..77ea8f59e0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -1,6 +1,5 @@ //! Host column migration planning while Python owns warehouse inspection. -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::replace; use crate::bindings::_helpers::planner_migrations::known_input_columns::KnownInputColumns; use crate::bindings::_helpers::planner_migrations::models::{ @@ -14,6 +13,8 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; /// Decide column renames and return the existing planning result model. #[pyfunction(name = "plan_column_migrations")] @@ -81,7 +82,7 @@ pub(crate) fn plan_column_migrations<'py>( let state = super::state_inspection::inspection(&runtime, None)?; let schemas = PySet::empty(py)?; for candidate in &candidates { - let schema = native_field(&model_reads::destination(&candidate.model)?, "schema")?; + let schema = native_field(&destination(&candidate.model)?, "schema")?; if !schema.is_none() { schemas.add(schema)?; } @@ -108,10 +109,7 @@ pub(crate) fn plan_column_migrations<'py>( &native_field(&state, "column_events")?, &inputs, )?; - let physical = mapping_get( - &columns.physical_relations, - model_reads::name_object(&candidate.model)?, - )?; + let physical = mapping_get(&columns.physical_relations, name_object(&candidate.model)?)?; let checked = super::column_entries::checked(&runtime, candidate, decided, &physical)?; for hint in super::column_entries::hints(candidate, &checked.entries)?.try_iter()? { hints.push(hint?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index ef732eccbf..fd9d453326 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -1,7 +1,5 @@ //! Convert existing planner contracts for origin compatibility checks. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::model_reads::inferred_rows; use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; use crate::bindings::_helpers::planner_migrations::models::{ CompatibilityCheck, MigrationCompatibilityResult, @@ -20,6 +18,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; fn config(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { config_value(model.py(), &model_values(model)?, key)?.extract() @@ -92,12 +92,9 @@ pub(super) fn check_migration_compatibility<'py>( ); } let relations = PyDict::new(py); - relations.set_item(model_reads::name_object(&model)?, relation)?; + relations.set_item(name_object(&model)?, relation)?; let columns = PyDict::new(py); - columns.set_item( - model_reads::name_object(&model)?, - check.origin_columns.clone(), - )?; + columns.set_item(name_object(&model)?, check.origin_columns.clone())?; let snapshot = new_record_with( py, RecordId::WarehouseSnapshot, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs index fb395b498c..c9164a2d4e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs @@ -1,6 +1,5 @@ //! Native migration discovery over existing inspection and fingerprint cache boundaries. -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::matching::{ match_migration_fingerprints, migration_destination_fingerprints, }; @@ -14,6 +13,7 @@ use pyo3::prelude::{PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple, PyTupleMethods}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use std::collections::BTreeSet; /// Manual and automatically discovered migration requests with their destination fingerprints. @@ -57,7 +57,7 @@ pub(super) fn discover_model_migrations<'py>( let declared = discovery_events::declared(&declarations, &state)?; let names = PyDict::new(py); for item in &models { - names.set_item(model_reads::name_object(item)?, item)?; + names.set_item(name_object(item)?, item)?; } let event_renames = discovery_events::renames(&runtime, &names, &state)?; for (name, origin) in declared.iter() { @@ -75,7 +75,7 @@ pub(super) fn discover_model_migrations<'py>( } let unbuilt = PyDict::new(py); for item in &models { - let name = model_reads::name_object(item)?; + let name = name_object(item)?; if !declared_names.contains(&name)? && call_with( &native_field(&state, "named_fingerprint")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs index 1b93b679da..dc9590f417 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs @@ -2,7 +2,6 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::config; use crate::bindings::_helpers::planner_migrations::models::ModelMigrationDeclaration; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -11,6 +10,7 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_move; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; pub(super) fn identity(item: &Bound<'_, PyAny>) -> PyResult { let materialized = config(item, "materialized")?; @@ -111,11 +111,11 @@ pub(super) fn resumed<'py>( let target = native_field(&native_field(runtime, "project")?, "effective_target_name")?; let events = native_field(state, "events")?; for item in models { - let name = model_reads::name_object(item)?; + let name = name_object(item)?; if !unbuilt.contains(&name)? || !(history(item)? || identity(item)?) { continue; } - let destination = model_reads::destination(item)?; + let destination = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(item)?; let event = newest(&events, &destination)?; if event.is_none() { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs index 4eaac7b808..ad16642e95 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs @@ -2,14 +2,9 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::graph::keys::native_key; use crate::bindings::_helpers::planner_identity::metadata::build_function_local_hashes; use crate::bindings::_helpers::planner_identity::metadata::build_model_version_identity_metadata_json; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; @@ -17,18 +12,23 @@ use sqlbuild_analysis::planner_migrations::main::discovery_order::discovery_orde use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_project::compiled_views::main::compiled_models::deps::deps; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::BTreeMap; pub(super) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult>> { let mut models: BTreeMap> = BTreeMap::new(); let mut dependencies: BTreeMap> = BTreeMap::new(); - for item in - project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? - { + for item in compiled_models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let item = item?; - let name: String = model_reads::name_object(&item)?.extract()?; + let name: String = name_object(&item)?.extract()?; let mut upstream: Vec = Vec::new(); - for dependency in model_reads::deps(&item)?.try_iter()? { + for dependency in deps(&item)?.try_iter()? { let dependency = dependency?; if native_field(&dependency, "resource_type")?.eq("model")? { upstream.push(native_field(&dependency, "name")?.extract()?); @@ -54,8 +54,8 @@ pub(super) fn triggered( let relevant = reachable_keys(&selected, &scope.upstream); let fingerprints = native_field(&native_field(snapshot, "fingerprints")?, "models")?; for item in models { - let name = model_reads::name_object(item)?; - if relevant.contains(&native_key(&model_reads::model_key(item)?)?) + let name = name_object(item)?; + if relevant.contains(&native_key(&model_key(item)?)?) && !fingerprints.contains(&name)? && !declared.contains(&name)? { @@ -74,7 +74,9 @@ pub(super) fn metadata<'py>( let functions = object( py, build_function_local_hashes( - &project_reads::functions(&project)?, + &sqlbuild_python_project::compiled_views::main::project::functions::functions( + &project, + )?, native_field(&project, "sql_analysis_dialect")? .extract::>()? .as_deref(), @@ -93,7 +95,7 @@ pub(super) fn metadata<'py>( let result = PyDict::new(py); for item in models { result.set_item( - model_reads::name_object(item)?, + name_object(item)?, object( py, build_model_version_identity_metadata_json( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs index a4138280ad..9af31f02f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs @@ -1,7 +1,5 @@ //! Live candidate inspection and automatic request materialization. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::graph::keys::native_key; use crate::bindings::_helpers::planner_migrations::column_objects::config; use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::discovery_events::{ @@ -10,7 +8,6 @@ use crate::bindings::_helpers::planner_migrations::discovery_events::{ use crate::bindings::_helpers::planner_migrations::models::{ ModelMigrationDeclaration, ModelMigrationRequest, }; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PySet, PySetMethods, PyString, PyTuple, @@ -22,6 +19,10 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::BTreeMap; fn location<'py>( @@ -118,13 +119,10 @@ pub(super) fn automatic<'py>( let scope = scope_data(scope)?; let mut requests: Vec> = Vec::new(); for item in models { - if !scope - .selected - .contains(&native_key(&model_reads::model_key(item)?)?) - { + if !scope.selected.contains(&native_key(&model_key(item)?)?) { continue; } - let name = model_reads::name_object(item)?; + let name = name_object(item)?; let event = matches.1.get_item(&name)?; let candidate = mapping_get(matches.0, &name)?; let origin = if let Some(event) = event { @@ -170,11 +168,8 @@ pub(super) fn warnings<'py>( let scope = scope_data(scope)?; let selected = PySet::empty(py)?; for item in models { - if scope - .selected - .contains(&native_key(&model_reads::model_key(item)?)?) - { - selected.add(model_reads::name_object(item)?)?; + if scope.selected.contains(&native_key(&model_key(item)?)?) { + selected.add(name_object(item)?)?; } } let ambiguous: BTreeMap = ambiguous.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs index 7a7b416f7d..53d4a0e078 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs @@ -1,7 +1,5 @@ //! Migration identity binding with unchanged metadata and hash format. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; @@ -10,6 +8,8 @@ use sqlbuild_analysis::planner_migrations::main::{migration_fingerprint, migrati use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::native_host::models::ModelLookup; use std::collections::BTreeMap; /// Hash normalized SQL and non-storage metadata through rename identities. @@ -103,7 +103,7 @@ pub(crate) fn with_migration_fingerprints<'py>( let fingerprint = call_with( &native_field(cache, "fingerprint")?, &[ - ("query_sql", model_reads::query_sql(&model)?), + ("query_sql", query_sql(&model)?), ("metadata_json", metadata), ("ref_identities", PyDict::new(py).into_any()), ("dialect", dialect.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs index ac808dabce..821272dfc1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs @@ -1,6 +1,5 @@ //! Fingerprint cache callbacks around native one-to-one migration matching. -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::models::{ FingerprintInputs, ModelMigrationRequest, }; @@ -12,6 +11,9 @@ use sqlbuild_analysis::planner_migrations::models::MigrationCandidate; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; use std::collections::{BTreeMap, BTreeSet}; fn variants<'py>( @@ -19,7 +21,7 @@ fn variants<'py>( model: &Bound<'py, PyAny>, maps: Bound<'py, PyAny>, ) -> PyResult> { - let name = model_reads::name_object(model)?; + let name = name_object(model)?; let metadata = inputs.metadata_jsons.get_item(name)?; let cache = native_field(&inputs.fingerprints, "fingerprint")?; let mut hashes: Vec = Vec::new(); @@ -27,7 +29,7 @@ fn variants<'py>( let hash: Option = call_with( &cache, &[ - ("query_sql", model_reads::query_sql(model)?), + ("query_sql", query_sql(model)?), ("metadata_json", metadata.clone()), ("ref_identities", map?), ("dialect", inputs.dialect.clone()), @@ -54,7 +56,7 @@ pub(super) fn match_migration_fingerprints<'py>( let mut by_name: BTreeMap> = BTreeMap::new(); let mut names: Vec = Vec::new(); for model in models { - let name: String = model_reads::model_name(model)?; + let name: String = model_name(model)?; if new_names.contains(&name) { names.push(name.clone()); } @@ -93,7 +95,7 @@ pub(super) fn migration_destination_fingerprints<'py>( let result = PyDict::new(py); for request in requests { let model = &request.model; - let name = model_reads::name_object(model)?; + let name = name_object(model)?; if mapping_get(&inputs.metadata_jsons, &name)?.is_none() { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs index ccafab948a..9e7f22bb28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs @@ -2,7 +2,6 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::{lower, model, replace, text}; use crate::bindings::_helpers::planner_migrations::compatibility::check_migration_compatibility; use crate::bindings::_helpers::planner_migrations::model_handover::{ @@ -23,6 +22,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_move; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) @@ -76,7 +76,10 @@ pub(super) fn decide<'py>( let py = runtime.py(); let request = decision_request.request; let item = request.model.clone(); - let destination = model_reads::destination(&item)?; + let destination = + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( + &item, + )?; let origin = decision_request.origin.clone(); let snapshot = decision_request.snapshot.clone(); let state = decision_request.state.clone(); @@ -102,7 +105,7 @@ pub(super) fn decide<'py>( py, RecordId::ModelMigrationPlanEntry, &[ - ("model_name", model_reads::name_object(&item)?), + ("model_name", name_object(&item)?), ("discovery", request.discovery.clone()), ( "decision", @@ -163,7 +166,7 @@ pub(super) fn decide<'py>( } let inherited = if recorded && !applies(&item, &newest, &snapshot)? { native_field(&native_field(&snapshot, "fingerprints")?, "models")? - .call_method1("get", (model_reads::name_object(&item)?,))? + .call_method1("get", (name_object(&item)?,))? } else { empty_version(&item, &fingerprint)? }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs index fdaea4ab1e..cd6e60dc9c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs @@ -1,15 +1,16 @@ //! Adapt immutable migration fingerprints without changing their persisted representation. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::fingerprints::query_hashes::compute_query_hash; use crate::bindings::_helpers::planner_migrations::column_objects::{replace, text}; use crate::bindings::_helpers::planner_migrations::constants::MIGRATION_MODEL_NAME_METADATA_KEY; -use crate::bindings::_helpers::planner_phases::objects::object; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::state_rows::main::event_rows::presumed_utc::presumed_utc; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::fingerprints::main::compute_query_hash::compute_query_hash; +use sqlbuild_python_project::planner_phases::main::object::object; pub(super) fn applies( model: &Bound<'_, PyAny>, @@ -17,7 +18,7 @@ pub(super) fn applies( snapshot: &Bound<'_, PyAny>, ) -> PyResult { let own = native_field(&native_field(snapshot, "fingerprints")?, "models")? - .call_method1("get", (model_reads::name_object(model)?,))?; + .call_method1("get", (name_object(model)?,))?; Ok(own.is_none() || presumed_utc(&native_field(&own, "ts")?)? < presumed_utc(&native_field(event, "created_at")?)?) @@ -45,18 +46,21 @@ pub(super) fn handover<'py>( if let Ok(payload) = payload.cast::() && payload.contains(key)? { - payload.set_item(key, model_reads::name_object(model)?)?; + payload.set_item(key, name_object(model)?)?; let kwargs = PyDict::new(py); kwargs.set_item("sort_keys", true)?; kwargs.set_item("separators", (",", ":"))?; kwargs.set_item("default", imported(py, "builtins", "str")?)?; metadata = native_field(&json, "dumps")?.call((payload,), Some(&kwargs))?; } - let destination = model_reads::destination(model)?; + let destination = + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( + model, + )?; replace( fingerprint, &[ - ("node_name", model_reads::name_object(model)?), + ("node_name", name_object(model)?), ("target_database", native_field(&destination, "database")?), ("target_schema", native_field(&destination, "schema")?), ("target_name", native_field(&destination, "name")?), @@ -88,14 +92,14 @@ pub(super) fn equivalent<'py>( replace( &handover, &[ - ("definition", model_reads::query_sql(model)?), + ("definition", query_sql(model)?), ( "definition_hash", object( py, compute_query_hash( py, - &model_reads::query_sql(model)?.extract::()?, + &query_sql(model)?.extract::()?, dialect.extract::>()?.as_deref(), )?, )?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 12d630e1e7..4f9a734749 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -1,7 +1,5 @@ //! Model migration orchestration over Python-owned live inspection. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_changes::renames::plan_reference_renames; use crate::bindings::_helpers::planner_migrations::column_objects::{config, replace}; use crate::bindings::_helpers::planner_migrations::discovery::discover_model_migrations; @@ -12,9 +10,7 @@ use crate::bindings::_helpers::planner_migrations::models::{ use crate::bindings::_helpers::planner_migrations::planning_origin::{ require_same_database, resolve_origin, }; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyString, PyTuple}; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; @@ -29,6 +25,10 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; const MIGRATE_FROM_KEY: &str = "migrate_from"; const MIGRATE_FORCE_KEY: &str = "migrate_force"; @@ -103,7 +103,7 @@ fn discover<'py>( let snapshot = phase.snapshot.clone(); let mut declared_requests: Vec> = Vec::new(); for item in - project_reads::models(&runtime_field(&runtime, |runtime| &runtime.project)?)?.try_iter()? + compiled_models(&runtime_field(&runtime, |runtime| &runtime.project)?)?.try_iter()? { if let Some(request) = declared(&item?)? { declared_requests.push(request); @@ -119,7 +119,7 @@ fn discover<'py>( for request in &declared_requests { let (origin_location, origin_model) = resolve_origin(request, &runtime, state)?; declarations.push(ModelMigrationDeclaration { - model_name: model_reads::name_object(&request.model)?, + model_name: name_object(&request.model)?, origin_location, origin_model, }); @@ -185,7 +185,7 @@ fn finish<'py>( let policy = super::planning_boundary::missing_origin_policy(&runtime)?; for (request, origin, origin_model) in resolved { let item = request.model.clone(); - let name = model_reads::name_object(&item)?; + let name = name_object(&item)?; let decided = super::model_entries::decide(&MigrationDecisionRequest { runtime: runtime.clone(), request, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs index d2bd1aaf94..eb374c54d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs @@ -1,6 +1,5 @@ //! Assemble new and resumed old-name view steps from recorded facts. -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::{model, text}; use crate::bindings::_helpers::planner_migrations::constants::{PENDING_ARCHIVE, PENDING_VIEW}; use crate::bindings::_helpers::planner_migrations::models::OldNamePlanner; @@ -9,7 +8,6 @@ use crate::bindings::_helpers::planner_migrations::old_name_objects::{ }; use crate::bindings::_helpers::planner_migrations::old_names::declared_column_aliases; use crate::bindings::_helpers::planner_migrations::old_names::old_name_view_skip_reason; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyBool, PyList, PyTuple}; @@ -21,6 +19,9 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::planner_phases::main::object::object; impl<'py> OldNamePlanner<'py> { pub(super) fn new( @@ -117,7 +118,7 @@ impl<'py> OldNamePlanner<'py> { &self, item: &Bound<'py, PyAny>, ) -> PyResult>> { - let Some(history) = self.newest(&relation(&model_reads::destination(item)?)?, None)? else { + let Some(history) = self.newest(&relation(&destination(item)?)?, None)? else { return Ok(None); }; if !native_field(&native_field(&history, "move")?, "destination_model")? @@ -327,7 +328,7 @@ impl<'py> OldNamePlanner<'py> { py, RecordId::OldNameViewPlanEntry, &[ - ("model_name", model_reads::name_object(item)?), + ("model_name", name_object(item)?), ( "origin", location(&self.runtime, &native_field(history, "old")?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs index 13face79d2..4f7811e34b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs @@ -2,7 +2,6 @@ use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::{config, model}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -12,6 +11,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::state_rows::main::migrations::old_name_retention::old_name_retention; +use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) @@ -44,7 +44,10 @@ pub(super) fn retention<'py>( project: &Bound<'py, PyAny>, ) -> PyResult> { old_name_retention( - &native_field(&model_reads::config(item)?, "values")?, + &native_field( + &sqlbuild_python_project::compiled_views::main::compiled_models::config::config(item)?, + "values", + )?, project, ) } @@ -72,7 +75,7 @@ pub(super) fn recorded_view<'py>( let created = native_field(history, "created")?; let mut reused = py.None().into_bound(py); for item in models { - if relation(&model_reads::destination(item)?)? + if relation(&destination(item)?)? .call_method1("matches", (native_field(history, "old")?,))? .is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index 6136b66601..4dbe2e9c79 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -2,15 +2,11 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; use crate::bindings::_helpers::planner_migrations::old_names::old_name_claim_errors; use crate::bindings::_helpers::planner_migrations::old_names::old_name_schemas_to_read; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PySet, PySetMethods, PyTuple}; use sqlbuild_analysis::planner_identity::constants::MODEL_RESOURCE; @@ -18,6 +14,12 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::HashMap; fn schemas<'py>( @@ -29,7 +31,7 @@ fn schemas<'py>( let snapshot = phase.snapshot.clone(); let needed = PySet::empty(py)?; for item in selected { - let schema = native_field(&model_reads::destination(item)?, "schema")?; + let schema = native_field(&destination(item)?, "schema")?; if !schema.is_none() { needed.add(schema)?; } @@ -48,7 +50,7 @@ fn schemas<'py>( if !include_project { for item in selected { let existing = native_field(&snapshot, "existing_relations")? - .call_method1("get", (model_reads::name_object(item)?,))?; + .call_method1("get", (name_object(item)?,))?; if !existing.is_none() && host_entry(py, AdapterHost::NormalizeRelationType)? .call1((native_field(&existing, "relation_type")?,))? @@ -80,12 +82,12 @@ pub(crate) fn plan_old_name_views<'py>( let runtime = phase.runtime.clone(); let scope = scope_data(&phase.scope)?; let project = runtime_field(&runtime, |runtime| &runtime.project)?; - let models: Vec> = project_reads::models(&project)? + let models: Vec> = compiled_models(&project)? .try_iter()? .collect::>()?; let mut by_name: HashMap> = HashMap::new(); for item in &models { - by_name.insert(model_reads::model_name(item)?, item.clone()); + by_name.insert(model_name(item)?, item.clone()); } let mut selected: Vec> = Vec::new(); for key in &scope.order { @@ -140,7 +142,7 @@ pub(crate) fn plan_old_name_views<'py>( } } for item in &selected { - if !covered.contains(model_reads::name_object(item)?)? + if !covered.contains(name_object(item)?)? && let Some(planned) = planner.pending_for(item)? { entries.push(planned); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs index 6f46d3a0d4..4a6a085759 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs @@ -3,10 +3,6 @@ use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::planner_migrations::column_objects::{native, text}; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::references::explicit_references::hard_coded_relation_remedy; -use crate::bindings::_helpers::references::explicit_references::old_name_reference_message; -use crate::bindings::_helpers::references::relations::match_project_relation; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -15,6 +11,10 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::references::main::hard_coded_relation_remedy::hard_coded_relation_remedy; +use sqlbuild_python_project::references::main::match_project_relation::match_project_relation; +use sqlbuild_python_project::references::main::old_name_reference_message::old_name_reference_message; fn old_relation<'py>( old: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs index 11c0f75d6f..1f5087138f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs @@ -2,7 +2,6 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::old_name_objects::relation; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -17,12 +16,14 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry; pub(crate) fn declared_column_aliases<'py>( py: Python<'py>, model: &Bound<'py, PyAny>, ) -> PyResult> { - let schema = model_reads::schema_entry(model)?; + let schema = schema_entry(model)?; let mut columns: Vec<(String, Option)> = Vec::new(); if !schema.is_none() { for column in native_field(&schema, "columns")?.try_iter()? { @@ -88,7 +89,7 @@ pub(crate) fn old_name_claim_errors<'py>( let mut errors: Vec> = Vec::new(); for model in arguments.value("selected")?.try_iter()? { let model = model?; - let name = model_reads::name_object(&model)?; + let name = name_object(&model)?; let existing = arguments .value("snapshot")? .getattr("existing_relations")? @@ -101,7 +102,7 @@ pub(crate) fn old_name_claim_errors<'py>( if !kind.eq("view")? { continue; } - let destination = relation(&model_reads::destination(&model)?)?; + let destination = relation(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?)?; for view in arguments.value("views")?.try_iter()? { let view = view?; if native_field(&view, "destination_model")?.eq(&name)? @@ -155,11 +156,11 @@ pub(crate) fn old_name_view_skip_reason<'py>( let mut reuser: Option = None; for model in arguments.value("models")?.try_iter()? { let model = model?; - if relation(&model_reads::destination(&model)?)? + if relation(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?)? .call_method1("matches", (&old,))? .is_truthy()? { - reuser = Some(model_reads::name_object(&model)?.extract()?); + reuser = Some(name_object(&model)?.extract()?); break; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs index f562594ae0..53fb78acba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs @@ -1,7 +1,5 @@ //! Resolve authored migration origins at the adapter and inspection boundaries. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_migrations::constants::QUALIFIED_RELATION_MAX_PARTS; use crate::bindings::_helpers::planner_migrations::models::ModelMigrationRequest; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; @@ -14,6 +12,8 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; pub(super) fn location<'py>( runtime: &Bound<'py, PyAny>, @@ -34,11 +34,9 @@ fn named<'py>( runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - for item in - project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? - { + for item in compiled_models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let item = item?; - if model_reads::name_object(&item)?.eq(name)? { + if name_object(&item)?.eq(name)? { return Ok((native_field(&item, "destination")?, name.clone())); } } @@ -73,8 +71,8 @@ fn input_error(py: Python<'_>, message: String) -> PyResult { pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { let project = runtime_field(runtime, |runtime| &runtime.project)?; let fallback = native_field(&project, "effective_target_database")?; - for item in project_reads::models(&project)?.try_iter()? { - let database = native_field(&model_reads::destination(&item?)?, "database")?; + for item in compiled_models(&project)?.try_iter()? { + let database = native_field(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&item?)?, "database")?; if !database.is_none() { return Ok(database); } @@ -85,10 +83,8 @@ pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult(runtime: &Bound<'py, PyAny>) -> PyResult> { let schemas = PySet::empty(runtime.py())?; - for item in - project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? - { - let schema = native_field(&model_reads::destination(&item?)?, "schema")?; + for item in compiled_models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { + let schema = native_field(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&item?)?, "schema")?; if !schema.is_none() { schemas.add(schema)?; } @@ -142,7 +138,10 @@ pub(super) fn resolve_origin<'py>( _ => super::column_objects::text(py, ""), }; let item = request.model.clone(); - let destination = model_reads::destination(&item)?; + let destination = + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( + &item, + )?; let mut parts: Vec> = Vec::new(); for part in raw.call_method1("split", (".",))?.try_iter()? { parts.push( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs index f47da108cf..820ea2335c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs @@ -1,7 +1,7 @@ //! Read-only warehouse evidence gathered once for all model migrations in one plan. -use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; use std::collections::{HashMap, HashSet}; use pyo3::prelude::{ @@ -25,9 +25,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; -use sqlbuild_python_front::state_rows::constants::COLUMN_KIND; -use sqlbuild_python_front::state_rows::constants::MODEL_KIND; -use sqlbuild_python_front::state_rows::constants::OLD_NAME_KIND; +use sqlbuild_python_front::state_rows::constants::{COLUMN_KIND, MODEL_KIND, OLD_NAME_KIND}; use sqlbuild_python_front::state_rows::main::migrations::layout::layout; use sqlbuild_python_front::state_rows::main::migrations::old_name_views::old_name_views; use sqlbuild_python_front::state_rows::main::migrations::read_events::read_events; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs index f0d4ba8962..9dbc69077b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs @@ -1,22 +1,23 @@ //! Selected-scope buildability validation before change planning. -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::graph::keys::native_key; -use crate::bindings::_helpers::planner_identity::graph::execution_origins; use crate::bindings::_helpers::planner_phases::constants::MISSING_UPSTREAM_CODE; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; -use crate::bindings::_helpers::planner_phases::objects::planner_error; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::source_tables::check_selected_source_tables_exist; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message; use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::planner_identity::main::execution_origins::execution_origins; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::resource_views::main::seed_reads::external::external; +use sqlbuild_python_project::resource_views::main::seed_reads::key::key; pub(crate) fn check_selected_scope_buildability<'py>( runtime: &Bound<'py, PyAny>, @@ -29,10 +30,10 @@ pub(crate) fn check_selected_scope_buildability<'py>( let selected_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; let scope = scope_data(&selected_scope)?; let mut satisfied_keys: std::collections::HashSet = std::collections::HashSet::new(); - for seed in project_reads::seeds(&project)?.try_iter()? { + for seed in seeds(&project)?.try_iter()? { let seed = seed?; - if seed_reads::external(&seed)?.is_truthy()? { - satisfied_keys.insert(GraphKey::from(&native_key(&seed_reads::key(&seed)?)?)); + if external(&seed)?.is_truthy()? { + satisfied_keys.insert(GraphKey::from(&native_key(&key(&seed)?)?)); } } let snapshot = warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs index f665c6b7b8..c009111ea3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs @@ -7,7 +7,7 @@ use pyo3::{PyClass, pyclass::boolean_struct::True}; use crate::bindings::_helpers::planner_phases::models::{ PlannerIdentityContext, PlannerScopeResolution, }; -use crate::bindings::_helpers::planner_scope::retained::project_scope; +use sqlbuild_python_project::planner_scope::main::project_scope::project_scope; #[pymethods] impl PlannerScopeResolution { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs index cacbbee94f..ef44a32502 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs @@ -1,9 +1,6 @@ //! Per-model full-refresh resolution and full-rebuild refusals. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_phases::constants::FULL_REFRESH_DISABLED_REBUILD_CODE; -use crate::bindings::_helpers::planner_phases::objects::{frozenset, planner_error, strings}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::types::PyBool; @@ -12,6 +9,12 @@ use sqlbuild_analysis::planner_warehouse::models::FullRebuildFacts; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; +use sqlbuild_python_project::planner_phases::main::strings::strings; const FULL_REFRESH_KEY: &str = "full_refresh"; @@ -37,10 +40,10 @@ pub(crate) fn effectively_full_refreshed_model_names<'py>( cli_full_refresh: bool, ) -> PyResult> { let mut names: Vec<_> = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; if resolve_model_full_refresh(&model, cli_full_refresh)? { - names.push(model_reads::name_object(&model)?); + names.push(name_object(&model)?); } } frozenset(project.py(), &names) @@ -79,7 +82,7 @@ pub(crate) fn check_full_rebuild_allowed( { return Ok(()); } - let name: String = model_reads::model_name(model)?; + let name: String = model_name(model)?; Err(planner_error( py, &format!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs index 948f3e9440..ca87274851 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/getters.rs @@ -6,9 +6,9 @@ use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use crate::bindings::_helpers::planner_phases::models::{ PlannerIdentityContext, PlannerRuntime, PlannerScopeResolution, PlannerWarehouseState, }; -use crate::bindings::_helpers::planner_scope::retained::RetainedScope; use sqlbuild_python_base::records::main::registry::class_of::class_of; use sqlbuild_python_base::records::models::GetterEntry; +use sqlbuild_python_project::planner_scope::models::RetainedScope; /// The getter readers of this module's native pyclasses. pub(crate) const GETTERS: &[GetterEntry] = &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs index 6d78148ab2..19ae42d660 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -1,14 +1,11 @@ //! Expected version identities and stale-warning change detection. -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_changes::detect::scope_model_changes; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::models::IdentityWork; use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; -use crate::bindings::_helpers::planner_phases::objects::native; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::sync::PyOnceLock; @@ -20,6 +17,10 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::project::functions::functions; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::{BTreeMap, HashSet}; static IDENTITY_WORK: PyOnceLock> = PyOnceLock::new(); @@ -53,10 +54,10 @@ fn version_identities<'py>( ) -> PyResult> { identity_work(project.py())?.identity_builds += 1; direct_model_version_identities( - (&project_reads::functions(project)?, &scope), + (&functions(project)?, &scope), native_field(project, "sql_analysis_dialect")?.extract()?, ( - &project_reads::seeds(project)?, + &seeds(project)?, &project.py().None().into_bound(project.py()), &native_field(project, "hook_functions")?, ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs index 0fd4cce629..c952ce457a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs @@ -9,7 +9,6 @@ pub(crate) mod full_refresh; pub(crate) mod getters; pub(crate) mod identities; pub(crate) mod models; -pub(crate) mod objects; pub(crate) mod output_assembly; pub(crate) mod registration; pub(crate) mod retention; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs deleted file mode 100644 index de5c5ebcdf..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs +++ /dev/null @@ -1,184 +0,0 @@ -//! Boundary helpers shared by the native planner phases. - -use sqlbuild_host::entries::models::HostId; - -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyFrozenSet, PyString, PyTuple}; -use pyo3::{BoundObject, IntoPyObject}; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::construct::enum_member::enum_member; -use sqlbuild_python_base::records::main::construct::error_class::error_class; -use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; -use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_front::model_config::main::config::config_text::config_text; -use sqlbuild_python_front::model_config::main::config::model_values::model_values; - -/// A Python `str`. -pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { - PyString::new(py, value).into_any() -} - -/// `None`, or the text. -pub(crate) fn optional_text<'py>(py: Python<'py>, value: Option<&str>) -> Bound<'py, PyAny> { - value.map_or_else(|| py.None().into_bound(py), |value| text(py, value)) -} - -/// The staying-Python class `class(**fields)`. -pub(crate) fn construct<'py>( - py: Python<'py>, - class: impl Into, - fields: &[(&str, Bound<'py, PyAny>)], -) -> PyResult> { - call_with(&host_entry(py, class)?, fields) -} - -/// A planner model dataclass. -pub(crate) fn planner_model<'py>( - py: Python<'py>, - name: RecordId, - fields: &[(&str, Bound<'py, PyAny>)], -) -> PyResult> { - new_record_with(py, name, fields) -} - -/// The native record `name(**fields)`. -pub(crate) fn native<'py>( - py: Python<'py>, - name: RecordId, - fields: &[(&str, Bound<'py, PyAny>)], -) -> PyResult> { - new_record_with(py, name, fields) -} - -/// `dataclasses.replace(value, **fields)`. -pub(crate) fn replace<'py>( - value: &Bound<'py, PyAny>, - fields: &[(&str, Bound<'py, PyAny>)], -) -> PyResult> { - replaced_keywords(value, &keywords(value.py(), fields)?) -} - -/// A staying-Python enum member from its value. -pub(crate) fn member<'py>( - py: Python<'py>, - class: impl Into, - value: &str, -) -> PyResult> { - host_entry(py, class)?.call1((value,)) -} - -/// A native string enum member from its value. -pub(crate) fn native_member<'py>( - py: Python<'py>, - class: RecordId, - value: &str, -) -> PyResult> { - new_record(py, class, (value,), None) -} - -/// `PlannerInputError(message, code=code, help=help)`. -pub(crate) fn planner_error( - py: Python<'_>, - message: &str, - (code, help): (Option<&str>, Option<&str>), -) -> PyErr { - let raised = error_class(py, ErrorId::PlannerInput).and_then(|class| { - let fields = keywords( - py, - &[ - ("code", optional_text(py, code)), - ("help", optional_text(py, help)), - ], - )?; - class.call((message,), Some(&fields)) - }); - match raised { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } -} - -/// A tuple of the items. -pub(crate) fn tuple<'py>( - py: Python<'py>, - items: Vec>, -) -> PyResult> { - Ok(PyTuple::new(py, items)?.into_any()) -} - -/// A frozenset of the items. -pub(crate) fn frozenset<'py>( - py: Python<'py>, - items: &[Bound<'py, PyAny>], -) -> PyResult> { - Ok(PyFrozenSet::new(py, items)?.into_any()) -} - -/// `sorted(items)` as a tuple of strings. -pub(crate) fn sorted_names<'py>( - py: Python<'py>, - mut names: Vec, -) -> PyResult> { - names.sort(); - tuple(py, names.iter().map(|name| text(py, name)).collect()) -} - -/// The strings of any Python iterable, in iteration order. -pub(crate) fn strings(values: &Bound<'_, PyAny>) -> PyResult> { - values - .try_iter()? - .map(|value| value?.extract::()) - .collect() -} - -/// The project nodes of `field` by name. -pub(crate) fn by_name<'py>( - project: &Bound<'py, PyAny>, - field: &str, -) -> PyResult> { - let nodes = pyo3::types::PyDict::new(project.py()); - for node in project_reads::collection(project, field)?.try_iter()? { - let node = node?; - pyo3::types::PyDictMethods::set_item(&nodes, resource_reads::name(&node)?, node)?; - } - Ok(nodes) -} - -/// A model config value as text, `None` when absent or not text. -pub(crate) fn config_str(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { - config_text(model.py(), &model_values(model)?, key) -} - -/// The compiled object key of the model `name`. -pub(crate) fn model_key<'py>( - py: Python<'py>, - name: &Bound<'py, PyAny>, -) -> PyResult> { - new_record_with( - py, - RecordId::CompiledObjectKey, - &[ - ( - "resource_type", - enum_member(py, RecordId::CompiledResourceType, "MODEL")?, - ), - ("name", name.clone()), - ], - ) -} - -/// A native helper's Rust result as a Python object. -pub(crate) fn object<'py, T>(py: Python<'py>, value: T) -> PyResult> -where - T: IntoPyObject<'py>, - PyErr: From, -{ - Ok(value.into_pyobject(py)?.into_bound().into_any()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 7755dfe65b..26493cd00c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -1,6 +1,5 @@ //! Plan output assembly: storage policies, warnings and direct-mode metadata. -use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::plan_output::plan_output_fields; use crate::bindings::_helpers::planner_identity::staleness::build_stale_out_of_selection_warnings; @@ -12,14 +11,9 @@ use crate::bindings::_helpers::planner_phases::models::{ PlanOutputContext, PlanOutputFields, PlanWarningContext, PlannedModelEntries, StoragePolicyContext, }; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_phases::objects::{ - native, replace, sorted_names, strings, text, tuple, -}; use crate::bindings::_helpers::planner_phases::retention::{plan_retention, plan_table_types}; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::source_freshness::source_freshness_unknown_warnings; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; @@ -30,6 +24,15 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::native_host::models::ModelLookup; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_phases::main::replace::replace; +use sqlbuild_python_project::planner_phases::main::sorted_names::sorted_names; +use sqlbuild_python_project::planner_phases::main::strings::strings; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; fn concat<'py>(py: Python<'py>, parts: &[Bound<'py, PyAny>]) -> PyResult> { let mut items: Vec<_> = Vec::new(); @@ -285,7 +288,7 @@ pub(crate) fn add_plan_warnings<'py>( /// The unselected models the selected scope reads, sorted. pub(crate) fn unselected_upstream_models(scopes: &Bound<'_, PyAny>) -> PyResult> { let selected = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; - let selected = crate::bindings::_helpers::planner_scope::retained::scope_data(&selected)?; + let selected = sqlbuild_python_project::planner_scope::main::scope_data::scope_data(&selected)?; let mut names: Vec = Vec::new(); for (key, deps) in &selected.upstream { if !selected.selected.contains(key) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs index fb4fc0679b..5a57fe9c6b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -4,38 +4,44 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; + module.add_class::()?; + module + .add_class::()?; + module + .add_class::()?; + module + .add_class::()?; module.add_function(wrap_pyfunction!( - super::scopes::resolve_planner_scopes, + crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes, module )?)?; module.add_function(wrap_pyfunction!( - super::identities::planner_identity_work_info, + crate::bindings::_helpers::planner_phases::identities::planner_identity_work_info, module )?)?; module.add_function(wrap_pyfunction!( - super::identities::detect_stale_warning_changes_of, + crate::bindings::_helpers::planner_phases::identities::detect_stale_warning_changes_of, module )?)?; module.add_function(wrap_pyfunction!( - super::full_refresh::describe_full_rebuild_cause, + crate::bindings::_helpers::planner_phases::full_refresh::describe_full_rebuild_cause, module )?)?; module.add_function(wrap_pyfunction!( - super::full_refresh::check_full_rebuild_allowed, + crate::bindings::_helpers::planner_phases::full_refresh::check_full_rebuild_allowed, module )?)?; module.add_function(wrap_pyfunction!( - super::retention::table_type_copy_name, + crate::bindings::_helpers::planner_phases::retention::table_type_copy_name, module )?)?; module.add_function(wrap_pyfunction!( - super::retention::plan_table_types, + crate::bindings::_helpers::planner_phases::retention::plan_table_types, + module + )?)?; + module.add_function(wrap_pyfunction!( + crate::bindings::_helpers::planner_phases::retention::plan_retention, module )?)?; - module.add_function(wrap_pyfunction!(super::retention::plan_retention, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index 3cf8f14a69..0db0b3022e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -2,19 +2,12 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_phases::constants::{ BIGQUERY_ADAPTER, DECREASE_WARNING, DOWNGRADE_HISTORY_DAYS, PERMANENT_TABLE_TYPE, RETENTION_DECREASE, RETENTION_INCREASE, RETENTION_MATCH, SNOWFLAKE_ADAPTER, TRANSIENT_TABLE_TYPE, }; -use crate::bindings::_helpers::planner_phases::objects::{ - construct, member, native, native_member, optional_text, planner_error, planner_model, replace, - sorted_names, text, tuple, -}; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -31,6 +24,21 @@ use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::typed_values::typed_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::planner_phases::main::construct::construct; +use sqlbuild_python_project::planner_phases::main::member::member; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_phases::main::native_member::native_member; +use sqlbuild_python_project::planner_phases::main::optional_text::optional_text; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; +use sqlbuild_python_project::planner_phases::main::planner_model::planner_model; +use sqlbuild_python_project::planner_phases::main::replace::replace; +use sqlbuild_python_project::planner_phases::main::sorted_names::sorted_names; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::{BTreeMap, BTreeSet}; const MATERIALIZED_KEY: &str = "materialized"; @@ -63,12 +71,12 @@ fn selected_models<'py>( let scope = scope_data(scope)?; let mut models: Vec<_> = Vec::new(); for model in - project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? + compiled_models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let model = model?; if scope .selected - .contains(&("model".to_owned(), model_reads::model_name(&model)?)) + .contains(&("model".to_owned(), model_name(&model)?)) { models.push(model); } @@ -94,12 +102,17 @@ pub(crate) fn plan_table_types<'py>( let mut declared: Vec<_> = Vec::new(); for model in &models { if native_field( - &native_field(&model_reads::config(model)?, "table_type")?, + &native_field( + &sqlbuild_python_project::compiled_views::main::compiled_models::config::config( + model, + )?, + "table_type", + )?, "declared", )? .is_truthy()? { - declared.push(model_reads::model_name(model)?); + declared.push(model_name(model)?); } } if !declared.is_empty() && !snowflake { @@ -136,7 +149,7 @@ pub(crate) fn plan_table_types<'py>( if !backed.is_truthy()? { continue; } - let relation = mapping_get(&relations, model_reads::name_object(&model)?)?; + let relation = mapping_get(&relations, name_object(&model)?)?; if relation.is_none() { continue; } @@ -152,7 +165,7 @@ pub(crate) fn plan_table_types<'py>( continue; } let downgrade = actual == Some(false) && desired_transient; - let destination = model_reads::destination(&model)?; + let destination = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?; let limit: i64 = adapter .call_method0("maximum_identifier_length")? .extract()?; @@ -161,7 +174,7 @@ pub(crate) fn plan_table_types<'py>( py, RecordId::TableTypePlanEntry, &[ - ("model_name", model_reads::name_object(&model)?), + ("model_name", name_object(&model)?), ("destination", destination), ("copy_name", text(py, ©_name(&target_name, limit))), ("desired_type", native_field(&native_field(&table_type, "value")?, "value")?), @@ -326,7 +339,7 @@ fn relation_retention<'py>( model: &Bound<'py, PyAny>, ) -> PyResult>> { let py = runtime.py(); - let name: String = model_reads::model_name(model)?; + let name: String = model_name(model)?; let config = native_field(model, "config")?; let retention = native_field(&config, "time_travel_retention")?; if native_field(&retention, "desired_days")?.is_none() @@ -339,7 +352,10 @@ fn relation_retention<'py>( )); } let desired_days: i64 = native_field(&retention, "desired_days")?.extract()?; - let destination = model_reads::destination(model)?; + let destination = + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( + model, + )?; let schema = native_field(&destination, "schema")?; if schema.is_none() { return Err(planner_error( @@ -352,7 +368,7 @@ fn relation_retention<'py>( py, AdapterHost::RetentionRequest, &[ - ("request_id", model_reads::name_object(model)?), + ("request_id", name_object(model)?), ( "scope", member(py, AdapterHost::RetentionScope, "relation")?, @@ -365,14 +381,14 @@ fn relation_retention<'py>( )?; let mut base = RetentionEntry { request: request.clone(), - model_names: tuple(py, vec![model_reads::name_object(model)?])?, + model_names: tuple(py, vec![name_object(model)?])?, state: None, source: native_field(&native_field(&retention, "source")?, "value")?, }; let snowflake = adapter_is(runtime, SNOWFLAKE_ADAPTER)?; let table_type = native_field(&native_field(&config, "table_type")?, "value")?; let relations = native_field(&native_field(warehouse, "snapshot")?, "existing_relations")?; - if !relations.contains(model_reads::name_object(model)?)? { + if !relations.contains(name_object(model)?)? { if snowflake && desired_days > 1 && table_type.eq(TRANSIENT_TABLE_TYPE)? { return Err(transient_error(py, &name, desired_days, "set ")); } @@ -383,7 +399,7 @@ fn relation_retention<'py>( flatten(&changes)?, )?]); } - let relation = relations.get_item(model_reads::name_object(model)?)?; + let relation = relations.get_item(name_object(model)?)?; let relation_transient = native_field(&relation, "is_transient")?; let lookup = call_with( &host_entry(py, AdapterHost::RelationLookup)?.getattr("key")?, @@ -477,7 +493,7 @@ fn bigquery_retention<'py>( { let model = model?; let retention = native_field(&native_field(&model, "config")?, "time_travel_retention")?; - let destination = model_reads::destination(&model)?; + let destination = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?; if native_field(&retention, "unmanaged")?.is_truthy()? || native_field(&retention, "desired_days")?.is_none() || native_field(&destination, "schema")?.is_none() @@ -494,7 +510,7 @@ fn bigquery_retention<'py>( } let mut scopes: BTreeSet<(String, String, Option)> = BTreeSet::new(); for model in models { - let destination = model_reads::destination(model)?; + let destination = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(model)?; let database: Option = native_field(&destination, "database")?.extract()?; let schema: String = native_field(&destination, "schema")?.str()?.extract()?; scopes.insert((database.clone().unwrap_or_default(), schema, database)); @@ -599,7 +615,12 @@ pub(crate) fn plan_retention<'py>( let py = runtime.py(); let mut models: Vec<_> = Vec::new(); for model in selected_models(runtime, scope)? { - let retention = native_field(&model_reads::config(&model)?, "time_travel_retention")?; + let retention = native_field( + &sqlbuild_python_project::compiled_views::main::compiled_models::config::config( + &model, + )?, + "time_travel_retention", + )?; if !native_field(&retention, "unmanaged")?.is_truthy()? && !native_field(&retention, "desired_days")?.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs index 9aec02133f..6e72967165 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs @@ -1,14 +1,14 @@ //! Resolve the selected, stale-warning and inspection scopes of one plan. use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; -use crate::bindings::_helpers::planner_scope::retained::scope_data; -use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; use pyo3::prelude::{Bound, PyAny, PyResult}; use pyo3::pyfunction; use pyo3::types::PyTuple; use sqlbuild_analysis::graph::main::reachable_keys::reachable_keys; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_scope::main::build_planner_scope::build_planner_scope; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; #[pyfunction] #[pyo3(signature = (*, project, selection, policies))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index d229588e9f..a682495b34 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -1,6 +1,5 @@ //! The warehouse inspection phase: snapshot, migrations and inspection relations in one pass. -use crate::bindings::_helpers::graph::views::compiled_graph_keys; use crate::bindings::_helpers::planner_entries::models::RelationsRequest; use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; use crate::bindings::_helpers::planner_migrations::column_planning::plan_column_migrations; @@ -13,15 +12,16 @@ use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::models::PlannerWarehouseState; -use crate::bindings::_helpers::planner_phases::objects::native; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; use crate::bindings::_helpers::planner_warehouse::snapshot::{SnapshotRequest, warehouse_snapshot}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::graph::main::project::compiled_graph_keys::compiled_graph_keys; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::time::Instant; fn progress(runtime: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index 30246d89b9..5cbd26a115 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -2,12 +2,9 @@ use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost}; -use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::plan_output::display::resolve_bounded_cursor_override; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::constants::CURSOR_START; use crate::bindings::_helpers::planner_resolution::cursor_bounds::compute_cursor_bounds; use crate::bindings::_helpers::planner_resolution::cursor_policies::{ @@ -34,6 +31,9 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_cursor_bound::config_cursor_bound; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::planner_phases::main::object::object; fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { let py: Python<'py> = model.py(); @@ -61,11 +61,14 @@ pub(crate) fn unbounded(py: Python<'_>) -> PyResult> { fn destination_exists(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> PyResult { let relation = native_field(snapshot, "existing_relations")? - .call_method1("get", (model_reads::name_object(model)?,))?; + .call_method1("get", (name_object(model)?,))?; if relation.is_none() { return Ok(false); } - let destination = model_reads::destination(model)?; + let destination = + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( + model, + )?; let key = call_with( &host_entry(model.py(), AdapterHost::RelationLookup)?.getattr("key")?, &[ @@ -81,7 +84,7 @@ fn history_redirected(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> let target = native_field(snapshot, "target_relation")?; Ok(!target.is_none() && target.ne(native_field( - &model_reads::destination(model)?, + &sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(model)?, "qualified_name", )?)?) } @@ -124,8 +127,8 @@ pub(crate) fn compute_model_cursor_bounds<'py>( return Ok(bounded); } let snapshot = &request.snapshot; - let mut cursor_snapshot = native_field(snapshot, "cursor_snapshots")? - .call_method1("get", (model_reads::name_object(&model)?,))?; + let mut cursor_snapshot = + native_field(snapshot, "cursor_snapshots")?.call_method1("get", (name_object(&model)?,))?; let roles = resolve_cursor_input_roles(&model)?; let producers = class_field(context, |c: &ModelPlanContext| { &c.runtime_cursor_producer_names @@ -152,7 +155,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( let refusal = object( py, empty_input_rebuild_refusal( - &model_reads::name_object(&model)?.extract::()?, + &name_object(&model)?.extract::()?, empty.extract::>()?, cursor_type.extract::>()?.as_deref(), ), @@ -178,7 +181,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( }; return Err(PyErr::from_value(unreadable_cursor_inputs_error( py, - &model_reads::name_object(&model)?.extract::()?, + &name_object(&model)?.extract::()?, cursor_type.extract::>()?.as_deref(), names.extract::>()?, )?)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs index 428d39f17b..b5d65ccb4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs @@ -1,6 +1,5 @@ //! Selected-model inspection and untyped override conversion. -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_resolution::values::{method, string}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -18,6 +17,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_text_is::config_text_is; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; const MATERIALIZED_KEY: &str = "materialized"; const CURSOR_KEY: &str = "cursor"; @@ -66,7 +66,7 @@ pub(crate) fn type_cursor_overrides<'py>( } for kind in [INTEGER_CURSOR, TIMESTAMP_CURSOR] { if config_text_is(py, &values, CURSOR_TYPE_KEY, kind)? { - models.push((model_reads::name_object(model)?.extract()?, kind.to_owned())); + models.push((name_object(model)?.extract()?, kind.to_owned())); } } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs index ae4eb4b540..605b995324 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -3,11 +3,9 @@ use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_python_base::records::main::construct::new_plain_error::new_plain_error; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::plan_output::display::resolve_bounded_cursor_override; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_phases::contexts::class_field; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::cursor_bounds::compute_cursor_bounds; use crate::bindings::_helpers::planner_resolution::cursor_bounds::duration; use crate::bindings::_helpers::planner_resolution::cursor_policies::{ @@ -46,6 +44,8 @@ use sqlbuild_python_front::cursor::models::CursorSession; use sqlbuild_python_front::model_config::main::config::config_cursor_bound::config_cursor_bound as typed_cursor_bound; use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::planner_phases::main::object::object; use std::cmp::Ordering; const INCREMENTAL: &str = "incremental"; @@ -362,8 +362,7 @@ fn model_cursor_snapshot<'py>( model: &Bound<'py, PyAny>, snapshot: &Bound<'py, PyAny>, ) -> PyResult> { - native_field(snapshot, "cursor_snapshots")? - .call_method1("get", (model_reads::name_object(model)?,)) + native_field(snapshot, "cursor_snapshots")?.call_method1("get", (name_object(model)?,)) } fn without_destination<'py>(cursor_snapshot: Bound<'py, PyAny>) -> PyResult> { @@ -556,7 +555,7 @@ pub(crate) fn model_cursor_overrides<'py>( } } let session = CursorSession::new(py)?; - let name = model_reads::name_object(model)?.str()?; + let name = name_object(model)?.str()?; let cursor_start = config_bound(model, "cursor_start")?; let cursor_end = config_bound(model, "cursor_end")?; if !resolved_start.is_none() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs index 6610d71335..01961f6424 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs @@ -1,18 +1,16 @@ //! Location maps retaining the original compiled Python destination objects. -use crate::bindings::_helpers::compiled_views::resource_reads; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_python_project::resource_views::main::resource_reads::destination::destination; +use sqlbuild_python_project::resource_views::main::resource_reads::name::name; fn locations<'py>(items: &Bound<'py, PyAny>) -> PyResult> { let result = PyDict::new(items.py()); for item in items.try_iter()? { let item = item?; - result.set_item( - resource_reads::name(&item)?, - resource_reads::destination(&item)?, - )?; + result.set_item(name(&item)?, destination(&item)?)?; } Ok(result) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs index 1e4e4779fc..faa31a2276 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs @@ -1,6 +1,5 @@ //! Maximum automatic-start recovery and evidence boundary. -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::cursor_bounds::apply_typed_cursor_replay_policy; use crate::bindings::_helpers::planner_resolution::cursor_bounds::{ optional_scalar, required_scalar, scalars, @@ -28,6 +27,7 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cursor::main::scalars::render_value::render_value; use sqlbuild_python_front::cursor::models::CursorSession; +use sqlbuild_python_project::planner_phases::main::object::object; use std::cmp::Ordering; #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index 30858a454f..453f203e19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -1,8 +1,5 @@ //! Model, function and SQL-hook orchestration through adapter-owned rendering. -use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_resolution::cursor_model::{ @@ -35,6 +32,10 @@ use sqlbuild_python_front::model_config::main::config::config_text_is::config_te use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; const APPEND_STRATEGY: &str = "append"; @@ -216,12 +217,9 @@ pub(crate) fn resolve_model_sql<'py>( cursor_type: cursor_type.clone(), lower_bound_inclusive: append_cursor_inclusive(model)?, }; - let mut sql = resolution.resolve( - &model_reads::query_sql(model)?.extract::()?, - &filtering, - )?; + let mut sql = resolution.resolve(&query_sql(model)?.extract::()?, &filtering)?; let (_, uses_intrinsics) = resolve_cursor_intrinsics(py, &sql, None, None)?; - let name: String = model_reads::model_name(model)?; + let name: String = model_name(model)?; if uses_intrinsics { let cause = match &request.full_rebuild_cause { Some(cause) => Some(cause.clone()), @@ -331,11 +329,10 @@ pub(crate) fn resolve_function_sql<'py>( function_locations: args.value("function_locations")?, }; let function = args.value("function")?; - let name: String = function_reads::name(&function)?.extract()?; - let sql: String = resolution.unbounded( - function_reads::body_sql(&function)?, - &format!("Function '{name}'"), - )?; + let name: String = + sqlbuild_python_project::resource_views::main::function_reads::name::name(&function)? + .extract()?; + let sql: String = resolution.unbounded(body_sql(&function)?, &format!("Function '{name}'"))?; Ok(PyString::new(py, &sql).into_any()) } @@ -358,7 +355,7 @@ pub(crate) fn model_hook_entries<'py>( ) -> PyResult> { let py: Python<'py> = model.py(); let values = model_values(model)?; - let name: String = model_reads::model_name(model)?; + let name: String = model_name(model)?; let mut hooks: Vec> = Vec::new(); for key in ["pre_hooks", "post_hooks"] { hooks.push(resolution.hook( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs index 8e3a4d36af..10bf24d5ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs @@ -5,7 +5,6 @@ use crate::bindings::_helpers::planner_resolution::values::{ cursor_wrap, filtering_of, method, string, }; use crate::bindings::_helpers::planner_strategy::values::input_error; -use crate::bindings::_helpers::references::source_relations::render_source_read; use pyo3::IntoPyObject; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -17,6 +16,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::type_system::main::types_equal::types_equal; +use sqlbuild_python_project::references::main::render_source_read::render_source_read; use sqlbuild_sqltext::resolution::main::column_name_key::column_name_key; use sqlbuild_sqltext::resolution::main::{ reference_spans::reference_spans, source_alias::source_alias, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs index a0ab6bfff6..d941f2657e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs @@ -5,7 +5,6 @@ use crate::bindings::_helpers::planner_scenario::constants::{ SCENARIO_PLAN_RELATION_COLLISION, }; use crate::bindings::_helpers::planner_scenario::values::{error, string}; -use crate::bindings::_helpers::relation_naming::fitting::fitted_logical_name; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_scenario::main::{hash_prefix, parse_artifact_name}; @@ -14,6 +13,7 @@ use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_v use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::relation_naming::main::fitted_logical_name::fitted_logical_name; const SCENARIO_ARTIFACT_LABEL: &str = "Scenario artifact"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index f07449c2cd..ab2e5b1c87 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -1,8 +1,5 @@ //! Assemble CLI scenario plans while retaining adapter and executor boundaries. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_hash_index; use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_relation_map; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; @@ -22,6 +19,9 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::planner_phases::main::object::object; #[pyfunction] #[pyo3(signature = (**kwargs))] @@ -142,14 +142,14 @@ fn namespace<'py>( ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py = project.py(); let models = PyDict::new(py); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; - models.set_item(model_reads::name_object(&model)?, model)?; + models.set_item(name_object(&model)?, model)?; } for field in ["target_model_names", "model_names"] { for name in native_field(graph, field)?.try_iter()? { if let Some(model) = models.get_item(name?)? { - let destination = model_reads::destination(&model)?; + let destination = sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?; return Ok(( native_field(&destination, "database")?, native_field(&destination, "schema")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs index 5375ac0efa..b26466437c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs @@ -1,7 +1,5 @@ //! Existing seed and function plan dataclass construction. -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_UNKNOWN_SEED; @@ -17,6 +15,9 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::by_attribute::by_attribute; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::resource_views::main::seed_reads::key::key; +use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; +use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; pub(super) fn seed_entries<'py>( project: &Bound<'py, PyAny>, @@ -24,7 +25,10 @@ pub(super) fn seed_entries<'py>( relations: &Bound<'py, PyAny>, ) -> PyResult> { let py = project.py(); - let seeds = by_attribute(&project_reads::seeds(project)?, "name")?; + let seeds = by_attribute( + &sqlbuild_python_project::compiled_views::main::project::seeds::seeds(project)?, + "name", + )?; let fixtures = native_field(graph, "seed_fixture_names")?; let mut result: Vec> = Vec::new(); for name in native_field(graph, "seed_names")?.try_iter()? { @@ -52,17 +56,17 @@ pub(super) fn seed_entries<'py>( py, RecordId::SeedPlanEntry, &[ - ("key", seed_reads::key(&seed)?), - ("name", seed_reads::name(&seed)?), - ("destination", destination), + ("key", key(&seed)?), ( - "file_path", - native_field(&seed_reads::seed_file(&seed)?, "file_path")?, + "name", + sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)?, ), + ("destination", destination), + ("file_path", native_field(&seed_file(&seed)?, "file_path")?), ("columns", extract_seed_columns(&seed)?.into_any()), ( "csv_settings", - native_field(&seed_reads::schema_entry(&seed)?, "csv_settings")?, + native_field(&schema_entry(&seed)?, "csv_settings")?, ), ], )?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index 2e69862e57..db3da0dc18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -1,13 +1,7 @@ //! Assemble scenario execution plans through existing planner and adapter APIs. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::records::compiled_model_record; -use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::planner_entries::model_entries::plan_model; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; -use crate::bindings::_helpers::planner_identity::graph::{execution_deps, execution_order}; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; use crate::bindings::_helpers::planner_scenario::constants::{ CURSOR_INTRINSIC_NAMES, CURSOR_TYPE_KEY, INCREMENTAL_MODE_KEY, INTEGER_CURSOR_TYPE, @@ -22,11 +16,6 @@ use crate::bindings::_helpers::planner_scenario::{ sql::{required_target, resolve_model_dbt_refs, resolve_sql, source_label}, values::{error, string}, }; -use crate::bindings::_helpers::planner_strategy::fixture_completion::complete_fixtures as complete_relation_fixtures; -use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; -use crate::bindings::_helpers::sql_test_glue::constants::{ - DEFAULT_INTEGER_END, DEFAULT_INTEGER_START, DEFAULT_TIMESTAMP_END, DEFAULT_TIMESTAMP_START, -}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::compiler::main::scenario_expected_column_message::scenario_expected_column_message; @@ -45,6 +34,20 @@ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_key use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_names::inferred_names; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::project::compiled_model_record::compiled_model_record; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::graph::models::KeyCache; +use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; +use sqlbuild_python_project::planner_identity::main::execution_order::execution_order; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_strategy::main::complete_fixtures::complete_fixtures as complete_relation_fixtures; +use sqlbuild_python_project::planner_strategy::main::planning_context::planning_context; +use sqlbuild_python_project::sql_test_glue::constants::{ + DEFAULT_INTEGER_END, DEFAULT_INTEGER_START, DEFAULT_TIMESTAMP_END, DEFAULT_TIMESTAMP_START, +}; #[pyfunction] #[pyo3(signature = (**kwargs))] @@ -99,9 +102,15 @@ pub(crate) fn build_scenario_execution_plan<'py>( } else { analysis.is_truthy()? }; - let models = by_attribute(&project_reads::models(&project)?, "name")?; - let functions = by_attribute(&project_reads::functions(&project)?, "key")?; - let locations = build_function_locations(&project_reads::functions(&project)?)?.into_any(); + let models = by_attribute(&compiled_models(&project)?, "name")?; + let functions = by_attribute( + &sqlbuild_python_project::compiled_views::main::project::functions::functions(&project)?, + "key", + )?; + let locations = build_function_locations( + &sqlbuild_python_project::compiled_views::main::project::functions::functions(&project)?, + )? + .into_any(); let upstream = execution_deps(&project)?; let ordered = execution_order(py, &upstream, &[])?; let ordered = KeyCache::new(py)?.keys(&ordered)?; @@ -137,13 +146,11 @@ pub(crate) fn build_scenario_execution_plan<'py>( let destination = required_target( &context.relations, "model_locations", - &model_reads::model_name(&model)?, + &model_name(&model)?, "model", )?; - let query = resolve_model_dbt_refs( - &model_reads::query_sql(&model)?.extract::()?, - &context.relations, - )?; + let query = + resolve_model_dbt_refs(&query_sql(&model)?.extract::()?, &context.relations)?; let cursor_window = scenario_cursor_window(&model)?; let record = compiled_model_record(&model)?; let changes = PyDict::new(py); @@ -274,9 +281,14 @@ fn model_context<'py>( ModelPlanContext { model_locations: native_field(&context.relations, "model_locations")?.unbind(), models_by_name: models.clone().into_any().into(), - functions_by_name: by_attribute(&project_reads::functions(project)?, "name")? - .into_any() - .unbind(), + functions_by_name: by_attribute( + &sqlbuild_python_project::compiled_views::main::project::functions::functions( + project, + )?, + "name", + )? + .into_any() + .unbind(), seed_locations: native_field(&context.relations, "seed_locations")?.unbind(), function_locations: locations.clone().unbind(), source_map: native_field(&context.relations, "source_map")?.unbind(), @@ -311,7 +323,7 @@ fn compared_columns( ) -> PyResult>> { let py = scenario.py(); let model_columns: Option> = match model { - Some(model) => model_reads::inferred_names(&model)?, + Some(model) => inferred_names(&model)?, None => None, }; let dialect: Option = @@ -402,7 +414,7 @@ fn expectations<'py>( /// The SQL-test wide window for a non-microbatch model whose SQL reads the cursor intrinsics. fn scenario_cursor_window<'py>(model: &Bound<'py, PyAny>) -> PyResult> { let py = model.py(); - let query: String = model_reads::query_sql(model)?.extract()?; + let query: String = query_sql(model)?.extract()?; let values = model_values(model)?; let microbatch: bool = config_value(py, &values, INCREMENTAL_MODE_KEY)?.eq(MICROBATCH_INCREMENTAL_MODE)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs index 02b2e7d4e5..e369c1a279 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs @@ -1,10 +1,5 @@ //! Convert project views into pure scenario dependency facts. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; use crate::bindings::_helpers::planner_scenario::values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; @@ -15,6 +10,12 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; +use sqlbuild_python_project::compiled_views::main::project::sources::sources; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::graph::models::KeyCache; #[pyfunction] #[pyo3(signature = (*, scenario, project, sql_analysis_enabled=true, sql_analysis_dialect=None))] @@ -37,25 +38,25 @@ pub(crate) fn plan_scenario_graph<'py>( dbt_refs: native_field(scenario, "dbt_ref_fixture_names")?.extract()?, ..ScenarioGraphInput::default() }; - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; let deps = native_field(&model, "deps")? .try_iter()? .map(|dep| native_key(&dep?)) .collect::>()?; - input - .models - .insert(model_reads::name_object(&model)?.extract()?, deps); + input.models.insert(name_object(&model)?.extract()?, deps); } - for source in project_reads::sources(project)?.try_iter()? { - input - .project_sources - .insert(source_reads::name(&source?)?.extract()?); + for source in sources(project)?.try_iter()? { + input.project_sources.insert( + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source?)? + .extract()?, + ); } - for seed in project_reads::seeds(project)?.try_iter()? { - input - .project_seeds - .insert(seed_reads::name(&seed?)?.extract()?); + for seed in seeds(project)?.try_iter()? { + input.project_seeds.insert( + sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed?)? + .extract()?, + ); } let output = graph(&input); input.targets.sort(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs index fe958d78cd..02f6055a68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs @@ -1,7 +1,5 @@ //! Build scenario relation locations through adapter qualification callbacks. -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::source_reads; use crate::bindings::_helpers::planner_scenario::constants::{REF, SOURCE}; use crate::bindings::_helpers::planner_scenario::constants::{ SCENARIO_PLAN_INTERNAL, SCENARIO_PLAN_MISSING_RELATION_TARGET, @@ -16,6 +14,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; #[pyfunction(name = "build_scenario_relation_plan")] #[pyo3(signature = (**kwargs))] @@ -66,11 +65,14 @@ pub(crate) fn build_scenario_relation_plan<'py>( )?; } let sources = PyDict::new(py); - for source in project_reads::sources(&args.project)?.try_iter()? { + for source in + sqlbuild_python_project::compiled_views::main::project::sources::sources(&args.project)? + .try_iter()? + { let source = source?; sources.set_item( - source_reads::name(&source)?, - source_reads::source_entry(&source)?, + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)?, + source_entry(&source)?, )?; } let models = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs index b10b6a00fe..323632ffd2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs @@ -1,6 +1,5 @@ //! Resolve scenario SQL through the shared native scanner and adapter callbacks. -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_scenario::constants::{ SCENARIO_PLAN_MISSING_RELATION_TARGET, SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, }; @@ -8,7 +7,6 @@ use crate::bindings::_helpers::planner_scenario::{ models::SqlContext, values::{error, string}, }; -use crate::bindings::_helpers::references::source_relations::render_source_relation; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyStringMethods}; use sqlbuild_analysis::planner_scenario::{ @@ -18,6 +16,8 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; use sqlbuild_sqltext::sql_scan::models::Unclosed; pub(super) fn resolve_sql( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs index ad41a18f00..63ff52ed86 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs @@ -1,8 +1,3 @@ //! Planner scope Python boundary models. -pub(crate) mod loaders; pub(crate) mod registration; -pub(crate) mod retained; -pub(crate) mod scope; -pub(crate) mod sources; -pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/registration.rs index 6ba143a726..7508b63a1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/registration.rs @@ -1,25 +1,16 @@ //! Register planner scope and loader entry points. -use crate::bindings::_helpers::planner_scope::{loaders, sources}; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::wrap_pyfunction; +use sqlbuild_python_project::planner_scope::main::build_intermediate_source_map_function::build_intermediate_source_map_function; +use sqlbuild_python_project::planner_scope::main::build_source_load_entries_function::build_source_load_entries_function; +use sqlbuild_python_project::planner_scope::main::build_source_load_map_function::build_source_load_map_function; +use sqlbuild_python_project::planner_scope::main::expand_selected_loader_dependencies_function::expand_selected_loader_dependencies_function; +use sqlbuild_python_project::planner_scope::main::upstream_loader_dependency_names_function::upstream_loader_dependency_names_function; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - loaders::upstream_loader_dependency_names, - module - )?)?; - module.add_function(wrap_pyfunction!( - loaders::build_intermediate_source_map, - module - )?)?; - module.add_function(wrap_pyfunction!( - loaders::expand_selected_loader_dependencies, - module - )?)?; - module.add_function(wrap_pyfunction!(sources::build_source_load_map, module)?)?; - module.add_function(wrap_pyfunction!( - sources::build_source_load_entries, - module - )?) + module.add_function(upstream_loader_dependency_names_function(module)?)?; + module.add_function(build_intermediate_source_map_function(module)?)?; + module.add_function(expand_selected_loader_dependencies_function(module)?)?; + module.add_function(build_source_load_map_function(module)?)?; + module.add_function(build_source_load_entries_function(module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/retained.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/retained.rs deleted file mode 100644 index 737e6c0694..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/retained.rs +++ /dev/null @@ -1,199 +0,0 @@ -//! Native planner scope ownership, with projection only at public record boundaries. - -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; - -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::pyclass; -use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use sqlbuild_analysis::assembly::project::types::ObjectKey; - -use crate::bindings::_helpers::graph::keys::KeyCache; -use crate::bindings::_helpers::graph::views::edge_dict; -use crate::bindings::_helpers::native_host::model_lookup::ModelHandle; -use crate::bindings::_helpers::planner_scope::values::{edges, keys}; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field; -use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; -use sqlbuild_python_base::records::models::NativeRecord; -use sqlbuild_python_base::records::types::RecordId; - -/// The graph and selection used throughout planning; model values remain compiled-resource views. -#[derive(Debug)] -pub(crate) struct ScopeData { - pub(crate) upstream: Vec<(ObjectKey, Vec)>, - pub(crate) downstream: Vec<(ObjectKey, Vec)>, - pub(crate) names: Vec<(String, ObjectKey)>, - pub(crate) models: HashMap, - pub(crate) model_order: Vec, - pub(crate) selected: HashSet, - pub(crate) order: Vec, - pub(crate) user_selected: HashSet, - pub(crate) python_reads: HashSet, - pub(crate) tests: Py, - pub(crate) load_sources: bool, - pub(crate) end_audit_sources: HashSet, -} - -/// Internal carrier, never returned in place of a supported public PlannerScope record. -#[pyclass(frozen)] -pub(crate) struct RetainedScope { - pub(crate) data: Arc, - projection: PyOnceLock>, -} - -impl ScopeData { - pub(crate) fn object(self, py: Python<'_>) -> PyResult> { - Ok(Bound::new( - py, - RetainedScope { - data: Arc::new(self), - projection: PyOnceLock::new(), - }, - )? - .into_any()) - } - - pub(crate) fn model<'py>(&self, py: Python<'py>, name: &str) -> Option> { - self.models.get(name).map(|model| model.bind(py)) - } - - pub(crate) fn with_selected(&self, py: Python<'_>, selected: HashSet) -> Self { - Self { - upstream: self.upstream.clone(), - downstream: self.downstream.clone(), - names: self.names.clone(), - model_order: self.model_order.clone(), - models: self - .models - .iter() - .map(|(name, model)| (name.clone(), model.clone_ref(py))) - .collect(), - selected, - order: self.order.clone(), - user_selected: self.user_selected.clone(), - python_reads: self.python_reads.clone(), - tests: self.tests.clone_ref(py), - load_sources: self.load_sources, - end_audit_sources: self.end_audit_sources.clone(), - } - } - - /// Public field projection; internal graph consumers read the native fields above instead. - pub(crate) fn field<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { - let cache = KeyCache::new(py)?; - let key_set = |keys: &HashSet| -> PyResult> { - Ok( - PyFrozenSet::new(py, cache.keys(&keys.iter().cloned().collect::>())?)? - .into_any(), - ) - }; - match name { - "upstream_deps" => Ok(edge_dict(&cache, &self.upstream)?.into_any()), - "downstream_deps" => Ok(edge_dict(&cache, &self.downstream)?.into_any()), - "selected_keys" => key_set(&self.selected), - "user_selected_keys" => key_set(&self.user_selected), - "end_audit_source_keys" => key_set(&self.end_audit_sources), - "execution_order" => Ok(PyTuple::new(py, cache.keys(&self.order)?)?.into_any()), - "all_keys" => { - let values = PyDict::new(py); - for (name, key) in &self.names { - values.set_item(name, cache.key(key)?)?; - } - Ok(values.into_any()) - } - "models_by_name" => { - let values = PyDict::new(py); - for name in &self.model_order { - if let Some(model) = self.models.get(name) { - values.set_item(name, model.bind(py))?; - } - } - Ok(values.into_any()) - } - "python_read_source_names" => Ok(PyFrozenSet::new(py, &self.python_reads)?.into_any()), - "sql_test_selection" => Ok(self.tests.bind(py).clone().into_any()), - "load_sources" => self.load_sources.into_bound_py_any(py), - _ => Err(pyo3::exceptions::PyAttributeError::new_err(name.to_owned())), - } - } -} - -/// Read native ownership directly, or import a caller-authored public scope at the API edge. -pub(crate) fn scope_data(value: &Bound<'_, PyAny>) -> PyResult> { - if let Ok(scope) = value.cast::() { - return Ok(scope.get().data.clone()); - } - let mut models: HashMap = HashMap::new(); - let mut model_order: Vec = Vec::new(); - for item in mapping_items(&field(value, "models_by_name")?)?.try_iter()? { - let item = item?; - let name: String = item.get_item(0)?.extract()?; - model_order.push(name.clone()); - models.insert(name, ModelHandle::read(&item.get_item(1)?)); - } - Ok(Arc::new(ScopeData { - upstream: edges(&field(value, "upstream_deps")?)?, - downstream: edges(&field(value, "downstream_deps")?)?, - names: mapping_items(&field(value, "all_keys")?)? - .try_iter()? - .map(|item| { - let item = item?; - Ok(( - item.get_item(0)?.extract()?, - crate::bindings::_helpers::graph::keys::native_key(&item.get_item(1)?)?, - )) - }) - .collect::>()?, - models, - model_order, - selected: keys(&field(value, "selected_keys")?)?.into_iter().collect(), - order: keys(&field(value, "execution_order")?)?, - user_selected: keys(&field(value, "user_selected_keys")?)? - .into_iter() - .collect(), - python_reads: field(value, "python_read_source_names")?.extract()?, - tests: field(value, "sql_test_selection")?.extract()?, - load_sources: field(value, "load_sources")?.is_truthy()?, - end_audit_sources: keys(&field(value, "end_audit_source_keys")?)? - .into_iter() - .collect(), - })) -} - -/// Preserve public record identity and dataclass behavior on explicit projection. -pub(crate) fn project_scope<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - let Ok(scope) = value.cast::() else { - return Ok(value.clone()); - }; - scope - .get() - .projection - .get_or_try_init(value.py(), || { - let names = [ - "upstream_deps", - "downstream_deps", - "all_keys", - "models_by_name", - "selected_keys", - "execution_order", - "user_selected_keys", - "python_read_source_names", - "sql_test_selection", - "load_sources", - "end_audit_source_keys", - ]; - let fields = names - .iter() - .map(|name| Ok((*name, scope.get().data.field(value.py(), name)?))) - .collect::>>()?; - Ok::<_, pyo3::PyErr>( - new_record_with(value.py(), RecordId::PlannerScope, &fields)? - .cast_into::() - .map(Bound::unbind)?, - ) - }) - .map(|value| value.bind(scope.py()).clone().into_any()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/mod.rs index 0097ebb362..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/mod.rs @@ -1,5 +1 @@ -pub(crate) mod carrier; -pub(crate) mod model_selectors; pub(crate) mod registration; -pub(crate) mod selectors; -pub(crate) mod unit_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs deleted file mode 100644 index 136cac00ac..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs +++ /dev/null @@ -1,120 +0,0 @@ -//! `+` marker splitting and SQLBuild-owned model selectors for dbt interop. - -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::PyBool; -use pyo3::types::{PyList, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::planner_selection::main::split_selector_expansion::split_selector_expansion as split; -use sqlbuild_analysis::planner_selection::main::sqlbuild_model_selector_names::sqlbuild_model_selector_names; -use sqlbuild_analysis::planner_selection::models::{ - ModelSelectorNames, SelectableModel, SelectorExpansion, -}; - -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::planner_selection::carrier::Carrier; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; -use sqlbuild_python_front::model_config::main::config::config_value::config_value; -use sqlbuild_python_front::model_config::main::config::model_values::model_values; - -const TAGS_KEY: &str = "tags"; - -const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; - -fn shared_input_error(carrier: &Carrier<'_>, message: &str) -> PyErr { - let raised = imported(carrier.py(), SHARED_EXCEPTIONS_MODULE, "SharedInputError") - .and_then(|class| class.call1((carrier.back(message)?,))); - match raised { - Ok(instance) => PyErr::from_value(instance), - Err(failure) => failure, - } -} - -/// Split `+selector+` marker syntax from the selector core without resolving the core. -#[pyfunction] -pub(crate) fn split_selector_expansion<'py>( - raw: &Bound<'py, PyString>, -) -> PyResult> { - let py: Python<'py> = raw.py(); - let carrier: Carrier<'py> = Carrier::single(raw)?; - let expansion: SelectorExpansion = - split(&carrier.send(raw)?).map_err(|message| shared_input_error(&carrier, &message))?; - new_record_with( - py, - RecordId::SelectorExpansion, - &[ - ("core", carrier.back(&expansion.core)?.into_any()), - ( - "upstream", - PyBool::new(py, expansion.upstream).to_owned().into_any(), - ), - ( - "downstream", - expansion - .downstream - .into_pyobject(py)? - .to_owned() - .into_any(), - ), - ], - ) -} - -fn model_tags(model: &Bound<'_, PyAny>) -> PyResult> { - let tags: Bound<'_, PyAny> = config_value(model.py(), &model_values(model)?, TAGS_KEY)?; - if !tags.is_instance_of::() && !tags.is_instance_of::() { - return Ok(Vec::new()); - } - tags.try_iter()?.map(|tag| tag?.str()?.extract()).collect() -} - -fn selectable_model(model: &Bound<'_, PyAny>) -> PyResult { - Ok(SelectableModel { - name: native_field(model, "name")?.extract()?, - tags: model_tags(model)?, - folder: native_field(&native_field(model, "relative_path")?, "parent")? - .call_method0("as_posix")? - .extract()?, - }) -} - -/// One SQLBuild-owned model selector core's model names, and its path selector when translated. -pub(crate) fn resolve_sqlbuild_model_selector_names<'py>( - project: &Bound<'py, PyAny>, - term: &Bound<'py, PyString>, -) -> PyResult> { - let py: Python<'py> = project.py(); - let models: Vec = project_reads::models(project)? - .try_iter()? - .map(|model| selectable_model(&model?)) - .collect::>()?; - let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(term), || { - Ok(models - .iter() - .flat_map(|model| [model.name.clone(), model.folder.clone()]) - .chain(models.iter().flat_map(|model| model.tags.clone())) - .collect()) - })?; - let resolved: ModelSelectorNames = sqlbuild_model_selector_names(&models, &carrier.send(term)?); - let translated: Option> = resolved - .translated - .as_deref() - .map(|path| carrier.back(path)) - .transpose()?; - PyTuple::new( - py, - [ - PyTuple::new(py, resolved.names)?.into_any(), - translated.into_pyobject(py)?.into_any(), - ], - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(split_selector_expansion, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/registration.rs index 46c95951b4..c8ba321fdf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/registration.rs @@ -2,9 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::planner_selection::model_selectors; +use sqlbuild_python_project::planner_selection::main::register_model_selectors::register_model_selectors; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - model_selectors::register(module)?; + register_model_selectors(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs deleted file mode 100644 index 6e6cf885b8..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs +++ /dev/null @@ -1,237 +0,0 @@ -//! Planner selector parsing and resolution over native project graphs, raising `PlannerInputError`. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyFrozenSet, PySetMethods, PyString}; -use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_analysis::graph::errors::SelectorError; -use sqlbuild_analysis::graph::main::match_selector::match_selector; -use sqlbuild_analysis::graph::main::parse_selector::parse_selector; -use sqlbuild_analysis::graph::main::resolve_selector_tokens::resolve_selector_tokens; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; -use sqlbuild_analysis::graph::models::{ParsedSelector, ProjectGraph}; - -use crate::bindings::_helpers::compiled_project::job_catalog::graph_with_jobs; -use crate::bindings::_helpers::graph::algorithms::closure; -use crate::bindings::_helpers::graph::keys::KeyCache; -use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; -use crate::bindings::_helpers::graph::views::{GraphViewMappings, graph_from_view_mappings}; -use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; -use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::types::RecordId; - -const VIEW_OPTIONS: &[&str] = &[ - "all_keys", - "upstream_deps", - "downstream_deps", - "tag_index", - "path_index", -]; - -/// The parsed selector as Python's `ParsedSelector` or `PathSelector`. -pub(crate) fn parsed_selector<'py>( - carrier: &Carrier<'py>, - parsed: ParsedSelector, -) -> PyResult> { - let py: Python<'py> = carrier.py(); - match parsed { - ParsedSelector::Kind { - kind, - value, - upstream, - downstream, - } => new_record_with( - py, - RecordId::ParsedSelector, - &[ - ("kind", enum_from_value(py, RecordId::SelectorKind, kind)?), - ("value", carrier.back(&value)?.into_any()), - ("upstream", PyBool::new(py, upstream).to_owned().into_any()), - ( - "downstream", - PyBool::new(py, downstream).to_owned().into_any(), - ), - ], - ), - ParsedSelector::Path { - start, - end, - upstream, - downstream, - } => new_record_with( - py, - RecordId::PathSelector, - &[ - ("start_name", carrier.back(&start)?.into_any()), - ("end_name", carrier.back(&end)?.into_any()), - ("upstream", PyBool::new(py, upstream).to_owned().into_any()), - ( - "downstream", - PyBool::new(py, downstream).to_owned().into_any(), - ), - ], - ), - } -} - -fn graph_names(graph: &ProjectGraph) -> Vec { - graph.names.iter().map(|(name, _)| name.clone()).collect() -} - -fn selected<'py>( - carrier: &Carrier<'py>, - outcome: Result, SelectorError>, -) -> PyResult> { - let keys: Vec = outcome.map_err(|error| carrier.error(error))?; - compiled_keys(carrier.py(), &keys) -} - -fn compiled_keys<'py>(py: Python<'py>, keys: &[ObjectKey]) -> PyResult> { - PyFrozenSet::new(py, KeyCache::new(py)?.keys(keys)?) -} - -fn resolved<'py>( - graph: &ProjectGraph, - (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - compiled_keys(select.py(), &resolve_graph_keys(graph, (select, exclude))?) -} - -/// Resolve graph keys for native planning without building a public frozenset. -pub(crate) fn resolve_graph_keys<'py>( - graph: &ProjectGraph, - (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { - let py: Python<'py> = select.py(); - let select: Vec> = py_strings(select)?; - let exclude: Vec> = py_strings(exclude)?; - let texts: Vec> = select.iter().chain(&exclude).cloned().collect(); - let carrier: Carrier<'py> = Carrier::new(py, &texts, || Ok(graph_names(graph)))?; - let (select, exclude) = (carrier.send_all(&select)?, carrier.send_all(&exclude)?); - let all: Vec = select.iter().chain(&exclude).cloned().collect(); - let graph = graph_with_jobs(py, graph, &all, &[])?; - resolve_selectors(&graph, &select, &exclude).map_err(|error| carrier.error(error)) -} - -fn view_graph(mappings: GraphViewMappings<'_, '_>) -> PyResult { - graph_from_view_mappings(mappings) -} - -/// Parse one selector token into Python's `ParsedSelector` or `PathSelector`. -pub(crate) fn parse_project_selector<'py>( - raw: &Bound<'py, PyString>, -) -> PyResult> { - let carrier: Carrier<'py> = Carrier::single(raw)?; - let parsed: ParsedSelector = - parse_selector(&carrier.send(raw)?).map_err(|error| carrier.error(error))?; - parsed_selector(&carrier, parsed) -} - -/// `--select` minus `--exclude` on a native graph, adding the functions selected models need. -pub(crate) fn resolve_graph_selection<'py>( - graph: PyRef<'py, NativeProjectGraph>, - select: &Bound<'py, PyAny>, - exclude: &Bound<'py, PyAny>, -) -> PyResult> { - resolved(graph.graph(), (select, exclude)) -} - -/// `--select` minus `--exclude` on a project graph, adding the functions selected models need. -pub(crate) fn select_project_graph<'py>( - graph: &Bound<'py, PyAny>, - select: &Bound<'py, PyAny>, - exclude: &Bound<'py, PyAny>, -) -> PyResult> { - let graph: &Bound<'py, CompiledProjectGraph> = graph.cast()?; - resolved( - graph.get().native(graph.py()).get().graph(), - (select, exclude), - ) -} - -/// Model, source, seed and function selectors resolved against compiled graph mappings. -pub(crate) fn resolve_project_selectors<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "resolve_project_selectors", - options, - (&[&["select", "exclude"], VIEW_OPTIONS].concat(), &[]), - )?; - let graph: NativeProjectGraph = arguments_graph(&arguments)?; - resolved( - graph.graph(), - (&arguments.value("select")?, &arguments.value("exclude")?), - ) -} - -/// The keys whitespace-separated selector tokens match, without required build resources. -pub(crate) fn resolve_project_selector_tokens<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "resolve_project_selector_tokens", - options, - (&[&["selectors"], VIEW_OPTIONS].concat(), &[]), - )?; - let graph: NativeProjectGraph = arguments_graph(&arguments)?; - let texts: Vec> = py_strings(&arguments.value("selectors")?)?; - let carrier: Carrier<'py> = Carrier::new(py, &texts, || Ok(graph_names(graph.graph())))?; - let selectors: Vec = carrier.send_all(&texts)?; - let graph = graph_with_jobs(py, graph.graph(), &selectors, &[])?; - selected(&carrier, resolve_selector_tokens(&graph, &selectors)) -} - -fn arguments_graph(arguments: &KeywordArguments<'_>) -> PyResult { - view_graph(GraphViewMappings { - all_keys: &arguments.value("all_keys")?, - upstream: &arguments.value("upstream_deps")?, - downstream: &arguments.value("downstream_deps")?, - tag_index: &arguments.value("tag_index")?, - path_index: &arguments.value("path_index")?, - }) -} - -/// The keys one parsed selector matches before `+` graph expansion. -pub(crate) fn match_project_selector<'py>( - parsed: &Bound<'py, PyAny>, - all_keys: &Bound<'py, PyAny>, - tag_index: &Bound<'py, PyAny>, - path_index: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = parsed.py(); - let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); - let graph: NativeProjectGraph = view_graph(GraphViewMappings { - all_keys, - upstream: &empty, - downstream: &empty, - tag_index, - path_index, - })?; - let value: Bound<'py, PyString> = native_field(parsed, "value")?.downcast_into()?; - let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(&value), || { - Ok(graph_names(graph.graph())) - })?; - let kind: String = native_field(parsed, "kind")?.str()?.extract()?; - let value: String = carrier.send(&value)?; - let graph = graph_with_jobs(py, graph.graph(), &[format!("{kind}:{value}")], &[])?; - selected( - &carrier, - match_selector(&graph, &kind, &value).map(|keys| keys.into_iter().collect()), - ) -} - -/// The upstream key closure of one compiled graph key. -pub(crate) fn expand_project_upstream_keys<'py>( - key: Bound<'py, PyAny>, - upstream_deps: &Bound<'py, PyAny>, -) -> PyResult> { - let visited = closure(vec![key], upstream_deps, (false, None))?; - PyFrozenSet::new(upstream_deps.py(), visited.iter()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs index 3e331b8bbf..6617268137 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -5,14 +5,6 @@ use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::models::GraphIndexes; use std::collections::HashSet; -use crate::bindings::_helpers::auditing::render::render_audit_sql; -use crate::bindings::_helpers::compiled_views::audit_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::graph::keys::native_key; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_scope::retained::scope_data; -use crate::bindings::_helpers::planner_scope::values::edges; use crate::bindings::_helpers::planner_strategy::audits::{ attachment, outside_target_lineage, resolve_effective_run_scope, }; @@ -23,17 +15,40 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PyString use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_front::attachments::main::assert_no_unresolved_sql_markers::assert_no_unresolved_sql_markers; +use sqlbuild_python_project::auditing::main::render_audit_sql::render_audit_sql; +use sqlbuild_python_project::compiled_views::main::project::audits::audits; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::graph::models::NativeProjectGraph; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_scope::main::edges::edges; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::resource_views::main::audit_reads::always_run::always_run; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_column_name::attached_column_name; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_kind::attached_target_kind; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_name::attached_target_name; +use sqlbuild_python_project::resource_views::main::audit_reads::definition_name::definition_name; +use sqlbuild_python_project::resource_views::main::audit_reads::description::description; +use sqlbuild_python_project::resource_views::main::audit_reads::evaluation_mode::evaluation_mode; +use sqlbuild_python_project::resource_views::main::audit_reads::evidence_limit::evidence_limit; +use sqlbuild_python_project::resource_views::main::audit_reads::evidence_sql::evidence_sql; +use sqlbuild_python_project::resource_views::main::audit_reads::measurement_contract::measurement_contract; +use sqlbuild_python_project::resource_views::main::audit_reads::meta::meta; +use sqlbuild_python_project::resource_views::main::audit_reads::minimum_samples::minimum_samples; +use sqlbuild_python_project::resource_views::main::audit_reads::run_scope::run_scope; +use sqlbuild_python_project::resource_views::main::audit_reads::scope_deps::scope_deps; +use sqlbuild_python_project::resource_views::main::audit_reads::sql_body::sql_body; +use sqlbuild_python_project::resource_views::main::audit_reads::thresholds::thresholds; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::Request; -use crate::bindings::_helpers::planner_strategy::constants::{ - AUDIT_TYPES_MODULE, COMPILE_TYPES_MODULE, MODEL_ATTACHMENT, SOURCE_ATTACHMENT, -}; -use crate::bindings::_helpers::planner_strategy::models::AuditPlanning; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::planner_strategy::constants::{ + AUDIT_TYPES_MODULE, COMPILE_TYPES_MODULE, MODEL_ATTACHMENT, SOURCE_ATTACHMENT, +}; +use sqlbuild_python_project::planner_strategy::models::AuditPlanning; /// One attribute read of a compiled audit. type AuditRead<'py> = fn(&Bound<'py, PyAny>) -> PyResult>; @@ -113,13 +128,15 @@ fn plan_audit<'py>( ) -> PyResult> { let py = audit.py(); let audit = audit.clone(); - let name: String = audit_reads::name(&audit)?.extract()?; + let name: String = + sqlbuild_python_project::resource_views::main::audit_reads::name::name(&audit)? + .extract()?; let resolved = render_query( request, - audit_reads::sql_body(&audit)?, + sql_body(&audit)?, &format!("audit '{name}' planned SQL"), )?; - let evidence = audit_reads::evidence_sql(&audit)?; + let evidence = evidence_sql(&audit)?; let evidence_resolved = if evidence.is_none() { py.None().into_bound(py) } else { @@ -142,33 +159,43 @@ fn plan_audit<'py>( } else { py.None().into_bound(py) }; - let requested = audit_enum(audit_reads::run_scope(&audit)?, "AuditRunScope", "final")?; - let severity = audit_enum(audit_reads::severity(&audit)?, "AuditSeverity", "warn")?; + let requested = audit_enum(run_scope(&audit)?, "AuditRunScope", "final")?; + let severity = audit_enum( + sqlbuild_python_project::resource_views::main::audit_reads::severity::severity(&audit)?, + "AuditSeverity", + "warn", + )?; let effective = resolve_effective_run_scope(&requested, &materialization)?; - let mut target_kind = audit_reads::attached_target_kind(&audit)?; + let mut target_kind = attached_target_kind(&audit)?; if target_kind.is_none() && (kind_text == MODEL_ATTACHMENT || kind_text == SOURCE_ATTACHMENT) { target_kind = imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind_text,))?; } let values = PyDict::new(py); let reads: [(&str, AuditRead<'py>); 12] = [ - ("key", audit_reads::key), - ("name", audit_reads::name), - ("definition_name", audit_reads::definition_name), - ("evaluation_mode", audit_reads::evaluation_mode), - ("thresholds", audit_reads::thresholds), - ("minimum_samples", audit_reads::minimum_samples), - ("evidence_limit", audit_reads::evidence_limit), - ("scope_deps", audit_reads::scope_deps), - ("attached_column_name", audit_reads::attached_column_name), - ("always_run", audit_reads::always_run), - ("description", audit_reads::description), - ("meta", audit_reads::meta), + ( + "key", + sqlbuild_python_project::resource_views::main::audit_reads::key::key, + ), + ( + "name", + sqlbuild_python_project::resource_views::main::audit_reads::name::name, + ), + ("definition_name", definition_name), + ("evaluation_mode", evaluation_mode), + ("thresholds", thresholds), + ("minimum_samples", minimum_samples), + ("evidence_limit", evidence_limit), + ("scope_deps", scope_deps), + ("attached_column_name", attached_column_name), + ("always_run", always_run), + ("description", description), + ("meta", meta), ]; for (field, read) in reads { values.set_item(field, read(&audit)?)?; } - let measurement = audit_reads::measurement_contract(&audit)?; + let measurement = measurement_contract(&audit)?; for field in ["value_column", "sample_count_column", "sample_unit"] { values.set_item( field, @@ -181,16 +208,13 @@ fn plan_audit<'py>( } for (field, value) in [ ("resolved_sql", resolved), - ("unresolved_sql", audit_reads::sql_body(&audit)?), + ("unresolved_sql", sql_body(&audit)?), ("attachment_kind", kind), ("severity", severity), ("requested_run_scope", requested), ("effective_run_scope", effective), ("evidence_resolved_sql", evidence_resolved), - ( - "evidence_unresolved_sql", - audit_reads::evidence_sql(&audit)?, - ), + ("evidence_unresolved_sql", evidence_sql(&audit)?), ("attached_target_kind", target_kind), ("attached_target_name", target), ( @@ -211,8 +235,8 @@ pub(crate) fn audit_is_selected( selected_keys: &HashSet, ) -> PyResult { let py = audit.py(); - let kind = audit_reads::attached_target_kind(audit)?; - let name = audit_reads::attached_target_name(audit)?; + let kind = attached_target_kind(audit)?; + let name = attached_target_name(audit)?; if !kind.is_none() && !name.is_none() { let target_value = native_field( &imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind,))?, @@ -221,7 +245,7 @@ pub(crate) fn audit_is_selected( let key = (target_value.extract()?, name.extract()?); return Ok(selected_keys.contains(&key)); } - for dependency in audit_reads::scope_deps(audit)?.try_iter()? { + for dependency in scope_deps(audit)?.try_iter()? { if selected_keys.contains(&native_key(&dependency?)?) { return Ok(true); } @@ -238,7 +262,7 @@ pub(crate) fn build_selected_audit_entries<'py>( let project = request.get("project")?; let scope = scope_data(&request.get("scope")?)?; let mut selected: Vec> = Vec::new(); - for audit in project_reads::audits(&project)?.try_iter()? { + for audit in audits(&project)?.try_iter()? { let audit = audit?; if audit_is_selected(&audit, &scope.selected)? { selected.push(audit); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs index 7602b752fc..3c4e27c50a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs @@ -15,13 +15,16 @@ use sqlbuild_analysis::planner_strategy::main::effective_run_scope::effective_ru use sqlbuild_analysis::planner_strategy::main::singular_attachment::singular_attachment; use sqlbuild_analysis::planner_strategy::models::{AuditAttachmentKind, AuditRunScope}; -use crate::bindings::_helpers::compiled_views::audit_reads; -use crate::bindings::_helpers::graph::keys::native_key; -use crate::bindings::_helpers::planner_scope::values::edges; use crate::bindings::_helpers::planner_strategy::values::{input_error, optional_string, text}; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::planner_scope::main::edges::edges; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_kind::attached_target_kind; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_name::attached_target_name; +use sqlbuild_python_project::resource_views::main::audit_reads::references::references; +use sqlbuild_python_project::resource_views::main::audit_reads::scope_deps::scope_deps; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; @@ -31,7 +34,7 @@ const FINAL_RUN_SCOPE: &str = "final"; /// The names of an audit's references of one kind, in order. fn reference_names(audit: &Bound<'_, PyAny>, kind: &str) -> PyResult> { let mut names: Vec = Vec::new(); - for reference in audit_reads::references(audit)?.try_iter()? { + for reference in references(audit)?.try_iter()? { let reference: Bound<'_, PyAny> = reference?; if text(&native_field(&reference, "ref_kind")?)? == kind { names.push(text(&native_field(&reference, "ref_name")?)?); @@ -77,7 +80,7 @@ pub(crate) fn attachment( graph: &ProjectGraph, ) -> PyResult<(String, Option)> { let py = audit.py(); - let attached_kind = audit_reads::attached_target_kind(audit)?; + let attached_kind = attached_target_kind(audit)?; let (kind, name): (String, Option) = if attached_kind.is_none() { let reached = RefCell::new(HashMap::new()); let reaches = |from: &str, to: &str| -> PyResult { @@ -100,14 +103,18 @@ pub(crate) fn attachment( )?; (attachment_value(kind).to_owned(), name) } else { - let name = audit_reads::attached_target_name(audit)?; + let name = attached_target_name(audit)?; if name.is_none() { return Err(input_error( py, ErrorId::PlannerInput, &format!( "audit '{}': attached audit is missing an attached target name", - text(&audit_reads::name(audit)?)? + text( + &sqlbuild_python_project::resource_views::main::audit_reads::name::name( + audit + )? + )? ), None, )); @@ -141,8 +148,8 @@ pub(crate) fn outside_target_lineage( graph: &ProjectGraph, ) -> PyResult { let py: Python<'_> = audit.py(); - let kind: Bound<'_, PyAny> = audit_reads::attached_target_kind(audit)?; - let name: Bound<'_, PyAny> = audit_reads::attached_target_name(audit)?; + let kind: Bound<'_, PyAny> = attached_target_kind(audit)?; + let name: Bound<'_, PyAny> = attached_target_name(audit)?; if kind.is_none() || name.is_none() { return Ok(false); } @@ -150,7 +157,7 @@ pub(crate) fn outside_target_lineage( text(&imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind,))?)?; let target = (kind, text(&name)?); let lineage = reachable_graph_keys(&[target], graph, EdgeDirection::Upstream); - for dependency in audit_reads::scope_deps(audit)?.try_iter()? { + for dependency in scope_deps(audit)?.try_iter()? { if !lineage.contains(&native_key(&dependency?)?) { return Ok(true); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs deleted file mode 100644 index 61a80e0b5e..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs +++ /dev/null @@ -1,31 +0,0 @@ -//! Audit plan boundary modules and attachment values. - -pub(crate) const AUDIT_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; -pub(crate) const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -pub(crate) const MODEL_ATTACHMENT: &str = "model"; -pub(crate) const SOURCE_ATTACHMENT: &str = "source"; -pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; -pub(crate) const MICROBATCH_MODE: &str = "microbatch"; -pub(crate) const CUSTOM_MATERIALIZATION: &str = "custom"; -pub(crate) const ENFORCED_CONTRACT: &str = "enforced"; -pub(crate) const TIMESTAMP_CURSOR: &str = "timestamp"; -pub(crate) const MODEL_REFERENCE: &str = "ref"; -pub(crate) const SEED_REFERENCE: &str = "seed"; -pub(crate) const SOURCE_REFERENCE: &str = "source"; -pub(crate) const DDL_SEPARATOR: &str = ";\n\n"; -/// The lineage confidence whose reads fixtures must provide. -pub(crate) const HIGH_CONFIDENCE: &str = "high"; -/// The lineage transform kind that passes an input column's type through. -pub(crate) const DIRECT_TRANSFORM: &str = "direct"; -/// Resource type values keying fixture relations. -pub(crate) const MODEL_REFERENCE_KIND: &str = "model"; -pub(crate) const SOURCE_REFERENCE_KIND: &str = "source"; -pub(crate) const SEED_REFERENCE_KIND: &str = "seed"; -/// The SQL analysis dialect whose adapter lays tables out by the physical storage keys. -pub(crate) const PHYSICAL_STORAGE_DIALECT: &str = "clickhouse"; -/// The error code for physical storage keys on an adapter without them. -pub(crate) const PHYSICAL_STORAGE_UNSUPPORTED_CODE: &str = "A011"; -/// The error code for `replace_key` on an adapter that cannot replace groups of rows. -pub(crate) const GROUP_REPLACEMENT_UNSUPPORTED_CODE: &str = "A012"; -/// Plan action values that create the model's table anew or leave it untouched. -pub(crate) const TABLE_RECREATING_ACTIONS: [&str; 2] = ["create_table", "skip"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs index 457d7b7b85..37b06bfb0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs @@ -7,18 +7,9 @@ use pyo3::types::{PyBool, PyDict, PyFrozenSet, PyString, PyTuple}; use sqlbuild_analysis::planner_strategy::main::cursor_input_column::cursor_input_column_failure; use sqlbuild_analysis::planner_strategy::models::{CursorInputColumnCheck, CursorInputColumnOwner}; -use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_identity::metadata::or_empty; -use crate::bindings::_helpers::planner_scope::retained::scope_data; -use crate::bindings::_helpers::planner_strategy::constants::{ - ENFORCED_CONTRACT, INCREMENTAL_MATERIALIZATION, MODEL_REFERENCE, SEED_REFERENCE, - SOURCE_REFERENCE, TIMESTAMP_CURSOR, -}; use crate::bindings::_helpers::planner_strategy::lineage::resolve_lineage_reference; -use crate::bindings::_helpers::planner_strategy::models::CursorInputLookups; use crate::bindings::_helpers::planner_strategy::values::{input_error, text}; -use crate::bindings::_helpers::references::source_relations::render_source_relation; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -32,6 +23,14 @@ use sqlbuild_python_front::model_config::main::config::config_cursor_bound::conf use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::planner_strategy::constants::{ + ENFORCED_CONTRACT, INCREMENTAL_MATERIALIZATION, MODEL_REFERENCE, SEED_REFERENCE, + SOURCE_REFERENCE, TIMESTAMP_CURSOR, +}; +use sqlbuild_python_project::planner_strategy::models::CursorInputLookups; +use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; /// Whether a model is incremental with a cursor column. fn incremental_cursor_flag( @@ -109,7 +108,10 @@ pub(crate) fn reject_unexposed_cursor_input_columns<'py>( let models_by_name = &lookups.models_by_name; let source_map: &Bound<'py, PyAny> = &lookups.source_map; let warehouse_columns: &Bound<'py, PyAny> = &lookups.source_warehouse_columns; - let model_name: String = model_reads::model_name(model)?; + let model_name: String = + sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name( + model, + )?; for role in ["filter_inputs", "watermark_inputs"] { for item in mapping_items(&native_field(&roles, role)?)?.try_iter()? { let (input_name, column): (String, String) = item?.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs index 30baee3ec5..40fdaa4647 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -9,10 +9,6 @@ use sqlbuild_analysis::planner_strategy::main::lineage_reference::lineage_refere use sqlbuild_analysis::planner_strategy::models::{ProducerFacts, ReferenceIdentity}; use sqlbuild_analysis::planner_strategy::types::ReferenceGraph; -use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; -use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; use crate::bindings::_helpers::planner_strategy::microbatches::{ CAP_ACTIONS, microbatch_limit_config, }; @@ -22,6 +18,10 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_front::model_config::main::config::config_text_is::config_text_is; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; +use sqlbuild_python_project::compiled_views::main::compiled_models::config::config; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::native_host::models::ModelLookup; const MICROBATCH_MODE: &str = "microbatch"; @@ -44,7 +44,9 @@ fn mapped_references<'py>( } fn references<'py>(owner: &Bound<'py, PyAny>) -> PyResult>> { - resource_reads::references(owner)?.try_iter()?.collect() + sqlbuild_python_project::resource_views::main::resource_reads::references::references(owner)? + .try_iter()? + .collect() } /// Whether a model's `incremental_mode` is microbatch. @@ -90,7 +92,7 @@ impl<'py> ReferenceGraph for ProjectLineage<'py> { fn producer_facts(&self, name: &str) -> PyResult { let producer = self.models_by_name.get(self.functions_by_name.py(), name)?; - let values: Bound<'py, PyAny> = native_field(&model_reads::config(&producer)?, "values")?; + let values: Bound<'py, PyAny> = native_field(&config(&producer)?, "values")?; let capped: bool = match microbatch_limit_config(&values)?.1 { Some(action) => CAP_ACTIONS.contains(&text(&action)?.as_str()), None => false, @@ -131,7 +133,7 @@ pub(crate) fn resolve_lineage_reference<'py>( models_by_name, functions_by_name, }; - let name: String = model_reads::model_name(model)?; + let name: String = model_name(model)?; lineage_reference(&graph, (&name, references(model)?), input_name) .map_err(|failure| lineage_error(py, failure)) } @@ -154,7 +156,7 @@ pub(crate) fn reject_capped_watermark_inputs<'py>( models_by_name, functions_by_name, }; - let name: String = model_reads::model_name(model)?; + let name: String = model_name(model)?; let inputs: Vec = watermark_names(model)?; match capped_watermark_input(&graph, (&name, references(model)?), inputs) { Ok(None) => Ok(()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs index cb4ead6819..9153633367 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs @@ -14,7 +14,6 @@ use sqlbuild_core::plan_display::errors::UnknownCursorGrain; use sqlbuild_core::plan_display::main::effective_timestamp_grain::effective_timestamp_grain; use sqlbuild_core::plan_display::models::TimestampGrainFacts; -use crate::bindings::_helpers::compiled_views::records::replace; use crate::bindings::_helpers::plan_output::display::resolve_effective_microbatch_batch_size; use crate::bindings::_helpers::planner_strategy::microbatches::{ CAP_ACTIONS, capped_microbatch_bounds, count_microbatches, @@ -23,6 +22,7 @@ use crate::bindings::_helpers::planner_strategy::values::{input_error, optional_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::project::replace::replace; const ERROR_ACTION: &str = "error"; const SKIP_ACTION: &str = "skip"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs index b29e2442b1..8176115560 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs @@ -1,23 +1,11 @@ pub(crate) mod audit_entries; pub(crate) mod audits; -pub(crate) mod constants; pub(crate) mod cursor_inputs; pub(crate) mod cursor_types; -pub(crate) mod fixture_completion; -pub(crate) mod fixture_context; -pub(crate) mod fixture_metadata; -pub(crate) mod fixture_preparation; -pub(crate) mod fixture_reads; -pub(crate) mod fixture_selection; -pub(crate) mod fixture_sql; -pub(crate) mod fixture_types; -pub(crate) mod fixture_values; pub(crate) mod lineage; pub(crate) mod microbatch_limits; pub(crate) mod microbatches; pub(crate) mod model_entries; -pub(crate) mod models; pub(crate) mod registration; pub(crate) mod strategy; -pub(crate) mod types; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 39ea87e0f1..6d08220ae0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -17,19 +17,11 @@ use sqlbuild_model_config::execution_limits::main::parse_execution_limit::parse_ use sqlbuild_model_config::execution_limits::models::ExecutionLimit; use sqlbuild_model_config::model_validation::constants::PHYSICAL_STORAGE_KEYS; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; -use crate::bindings::_helpers::planner_strategy::constants::{ - CUSTOM_MATERIALIZATION, DDL_SEPARATOR, ENFORCED_CONTRACT, GROUP_REPLACEMENT_UNSUPPORTED_CODE, - MICROBATCH_MODE, PHYSICAL_STORAGE_DIALECT, PHYSICAL_STORAGE_UNSUPPORTED_CODE, - TABLE_RECREATING_ACTIONS, -}; use crate::bindings::_helpers::planner_strategy::cursor_types::{ cursor_type_warning_record, warehouse_column_types, }; use crate::bindings::_helpers::planner_strategy::microbatch_limits::{replay_grain, unknown_grain}; use crate::bindings::_helpers::planner_strategy::microbatches::microbatch_limit_config; -use crate::bindings::_helpers::planner_strategy::models::ModelEntryDecisions; use crate::bindings::_helpers::planner_strategy::strategy::{ model_warning_records, resolve_schema_actions, warning_inputs, }; @@ -54,6 +46,18 @@ use sqlbuild_python_front::model_config::main::config::config_text_tuple::config use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; use sqlbuild_python_front::model_config::types::ConfigValues; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_rows::inferred_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::compiled_models::relative_path::relative_path; +use sqlbuild_python_project::planner_strategy::constants::{ + CUSTOM_MATERIALIZATION, DDL_SEPARATOR, ENFORCED_CONTRACT, GROUP_REPLACEMENT_UNSUPPORTED_CODE, + MICROBATCH_MODE, PHYSICAL_STORAGE_DIALECT, PHYSICAL_STORAGE_UNSUPPORTED_CODE, + TABLE_RECREATING_ACTIONS, +}; +use sqlbuild_python_project::planner_strategy::models::ModelEntryDecisions; /// Model config keys copied to the plan entry as optional strings, with their entry fields. const STRING_FIELDS: [(&str, &str); 15] = [ @@ -113,7 +117,10 @@ fn schema_columns<'py>( typed_only: bool, ) -> PyResult> { let py: Python<'py> = model.py(); - let entry: Bound<'py, PyAny> = resource_reads::schema_entry(model)?; + let entry: Bound<'py, PyAny> = + sqlbuild_python_project::resource_views::main::resource_reads::schema_entry::schema_entry( + model, + )?; if entry.is_none() { return Ok(PyTuple::empty(py)); } @@ -143,7 +150,7 @@ fn inferred_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult = model.py(); let class: Bound<'py, PyAny> = host_entry(py, AdapterHost::ColumnInfo)?; let mut columns: Vec> = Vec::new(); - for (name, column_type, _) in model_reads::inferred_rows(model)?.unwrap_or_default() { + for (name, column_type, _) in inferred_rows(model)?.unwrap_or_default() { if let Some(column_type) = column_type { columns.push(column_info( &class, @@ -196,7 +203,7 @@ fn on_schema_change<'py>(model: &Bound<'py, PyAny>) -> PyResult(model: &Bound<'py, PyAny>) -> PyResult(model: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = model.py(); - let entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; + let entry: Bound<'py, PyAny> = + sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( + model, + )?; if !entry.is_none() { let enforcement: Bound<'py, PyAny> = native_field(&entry, "type_enforcement")?; if !enforcement.is_none() { @@ -263,7 +273,10 @@ fn logical_ddl<'py>( let py: Python<'py> = request.model.py(); let adapter: &Bound<'py, PyAny> = &request.adapter; let model: &Bound<'py, PyAny> = &request.model; - let destination: Bound<'py, PyAny> = model_reads::destination(model)?; + let destination: Bound<'py, PyAny> = + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( + model, + )?; let qualified: Bound<'py, PyAny> = native_field(&destination, "qualified_name")?; let qualified: Bound<'py, PyAny> = if qualified.is_truthy()? { qualified @@ -326,7 +339,7 @@ fn logical_ddl<'py>( /// The output column names SQL analysis inferred for the model, when it inferred them. fn expected_output_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = model.py(); - let inferred: Bound<'py, PyAny> = model_reads::inferred_columns(model)?; + let inferred: Bound<'py, PyAny> = sqlbuild_python_project::compiled_views::main::compiled_models::inferred_columns::inferred_columns(model)?; if inferred.is_none() { return Ok(none(py)); } @@ -362,7 +375,7 @@ fn storage_fields<'py>( ErrorId::CompileInput, &format!( "model '{}' sets {}, but adapter '{}' has no physical table layout settings", - model_reads::model_name(&request.model)?, + model_name(&request.model)?, keys.join(", "), native_field(&request.adapter, "adapter_name")?.str()? ), @@ -412,7 +425,7 @@ fn model_warnings<'py>( ) -> PyResult> { let py: Python<'py> = request.model.py(); let model: &Bound<'py, PyAny> = &request.model; - let name: Bound<'py, PyAny> = model_reads::name_object(model)?; + let name: Bound<'py, PyAny> = name_object(model)?; let inputs = warning_inputs( &name, &request.materialization_type, @@ -593,7 +606,10 @@ fn contract_fields<'py>( mapping_get(values, "contract")?.eq(ENFORCED_CONTRACT)?, )?; entry.set_item("contract_columns", schema_columns(model, false)?)?; - let schema_entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; + let schema_entry: Bound<'py, PyAny> = + sqlbuild_python_project::compiled_views::main::compiled_models::schema_entry::schema_entry( + model, + )?; entry.set_item( "contract_dynamic_columns", if schema_entry.is_none() { @@ -651,7 +667,7 @@ pub(crate) fn assemble_model_plan_entry<'py>( let values: ConfigValues<'py> = model_values(model)?; let change: &Bound<'py, PyAny> = &request.change_result; let snapshot: &Bound<'py, PyAny> = &request.snapshot; - let name: Bound<'py, PyAny> = model_reads::name_object(model)?; + let name: Bound<'py, PyAny> = name_object(model)?; let policy: Bound<'py, PyAny> = on_schema_change(model)?; let schema_actions: Bound<'py, PyAny> = resolve_schema_actions(&native_field(change, "schema_findings")?, &policy)?.into_any(); @@ -666,7 +682,7 @@ pub(crate) fn assemble_model_plan_entry<'py>( let unique_key: Bound<'py, PyTuple> = string_list(&values.0, "unique_key")?; let replace_key: Bound<'py, PyTuple> = string_list(&values.0, "replace_key")?; if let Some(refusal) = group_replacement_refusal( - &model_reads::model_name(model)?, + &model_name(model)?, ( &native_field(&request.adapter, "adapter_name")? .str()? @@ -689,7 +705,12 @@ pub(crate) fn assemble_model_plan_entry<'py>( .unwrap_or_else(|| PyTuple::empty(py)); let table_type: Bound<'py, PyAny> = native_field( &native_field( - &native_field(&model_reads::config(model)?, "table_type")?, + &native_field( + &sqlbuild_python_project::compiled_views::main::compiled_models::config::config( + model, + )?, + "table_type", + )?, "value", )?, "value", @@ -705,11 +726,16 @@ pub(crate) fn assemble_model_plan_entry<'py>( (&warehouse_columns, ddl_cursor_bounds(request)?), )?; let entry: Bound<'py, PyDict> = PyDict::new(py); - entry.set_item("key", model_reads::model_key(model)?)?; + entry.set_item("key", model_key(model)?)?; entry.set_item("name", &name)?; - entry.set_item("relative_path", model_reads::relative_path(model)?)?; - entry.set_item("destination", model_reads::destination(model)?)?; - entry.set_item("fingerprint_query_sql", model_reads::query_sql(model)?)?; + entry.set_item("relative_path", relative_path(model)?)?; + entry.set_item( + "destination", + sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( + model, + )?, + )?; + entry.set_item("fingerprint_query_sql", query_sql(model)?)?; entry.set_item("logical_ddl", table_type_ddl(&ddl, &text(&table_type)?))?; passed_fields(request, &entry)?; for field in CHANGE_FIELDS { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs deleted file mode 100644 index 4f5a155c6e..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs +++ /dev/null @@ -1,110 +0,0 @@ -//! Extension-owned fixture analysis reuse and the project-wide fixture planning context. - -use std::collections::{BTreeSet, HashMap}; -use std::sync::{Arc, Mutex}; - -use pyo3::prelude::{Bound, Py}; -use pyo3::pyclass; -use pyo3::types::PyAny; -use sqlbuild_analysis::planner_fixtures::models::{ - FixtureColumn, FixtureReadCache, FixtureTypeProjection, -}; -use sqlbuild_analysis::required_columns::models::ColumnRead; - -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::native_host::model_lookup::ModelHandle; -use crate::bindings::_helpers::native_host::model_lookup::ModelLookup; -use crate::bindings::_helpers::planner_strategy::types::FixtureKey; - -/// Per-extension cache replacing the Python fixture resolver's LRU. -#[pyclass] -#[derive(Debug, Default)] -pub(crate) struct FixtureReadCacheHost { - pub(crate) cache: FixtureReadCache, - /// Fixture planning contexts built since the last reset. - pub(crate) context_builds: u64, - /// SQL-test model chain batches resolved since the last reset. - pub(crate) chain_resolutions: u64, -} - -/// One relation's known fixture columns and whether they form an authoritative shape. -#[derive(Clone, Debug)] -pub(crate) struct FixtureRelation { - pub(crate) columns: Vec, - pub(crate) authoritative: bool, -} - -/// One model's high-confidence lineage reads and direct type projections. -#[derive(Debug)] -pub(crate) struct ModelFixtureFacts { - pub(crate) reads: Vec, - pub(crate) projections: Vec, -} - -/// Project-wide fixture metadata shared across test and scenario planning. -#[pyclass(module = "sqlbuild._native", frozen)] -#[derive(Debug)] -pub(crate) struct RelationFixturePlanningContext { - pub(crate) models_by_name: HashMap, - pub(crate) relations: HashMap, - pub(crate) authoritative_columns: HashMap>, - pub(crate) expected_types: HashMap>, - pub(crate) fixture_inferences: Py, - pub(crate) model_facts: Mutex>>, -} - -/// The expected column types of one completion: its chain's propagated types over the context's. -#[derive(Debug)] -pub(crate) struct ExpectedTypes<'a> { - pub(crate) overlay: Vec<(FixtureKey, Vec<(String, String)>)>, - pub(crate) base: &'a HashMap>, -} - -/// One model's resolved plan decisions, assembled into its `ModelPlanEntry`. -#[derive(Debug)] -pub(crate) struct ModelEntryDecisions<'py> { - pub(crate) model: Bound<'py, PyAny>, - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) snapshot: Bound<'py, PyAny>, - pub(crate) change_result: Bound<'py, PyAny>, - pub(crate) action: Bound<'py, PyAny>, - pub(crate) reason: Bound<'py, PyAny>, - pub(crate) materialization_type: Bound<'py, PyAny>, - pub(crate) resolved_sql: Bound<'py, PyAny>, - pub(crate) sql_analysis_enabled: bool, - pub(crate) hooks: (Bound<'py, PyAny>, Bound<'py, PyAny>), - pub(crate) cursor_column: Bound<'py, PyAny>, - pub(crate) cursor_input_relations: Bound<'py, PyAny>, - pub(crate) runtime_owned_cursor_bounds: bool, - pub(crate) cursor_bounds: Bound<'py, PyAny>, - pub(crate) microbatch_range: Bound<'py, PyAny>, - pub(crate) empty_cursor_inputs: Bound<'py, PyAny>, - pub(crate) waiting_on_empty_inputs: bool, - pub(crate) backfill: Bound<'py, PyAny>, - pub(crate) cursor_overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), - pub(crate) cursor_configs: (Bound<'py, PyAny>, Bound<'py, PyAny>), - pub(crate) invocation_time: Bound<'py, PyAny>, -} - -/// The project lookups that resolve one model's cursor inputs. -#[derive(Debug)] -pub(crate) struct CursorInputLookups<'py> { - pub(crate) models_by_name: ModelLookup, - pub(crate) functions_by_name: Bound<'py, PyAny>, - pub(crate) source_map: Bound<'py, PyAny>, - pub(crate) source_warehouse_columns: Bound<'py, PyAny>, - pub(crate) model_locations: Bound<'py, PyAny>, - pub(crate) seed_locations: Bound<'py, PyAny>, - pub(crate) runtime_cursor_producer_names: Bound<'py, PyAny>, -} - -/// The relations and lineage that render and schedule selected audits. -#[derive(Debug)] -pub(crate) struct AuditPlanning<'py> { - pub(crate) model_locations: Bound<'py, PyAny>, - pub(crate) seed_locations: Bound<'py, PyAny>, - pub(crate) source_map: Bound<'py, PyAny>, - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) model_materializations: Bound<'py, PyAny>, - pub(crate) lineage: Py, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index fdbc656dbd..5ae6a39806 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -3,44 +3,45 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use pyo3::{prelude::PyModuleMethods, wrap_pyfunction}; -use crate::bindings::_helpers::planner_strategy::{ - audits, cursor_types, fixture_completion, microbatches, strategy, -}; +use crate::bindings::_helpers::planner_strategy::{audits, cursor_types, microbatches, strategy}; +use sqlbuild_python_project::planner_strategy::main::register_fixture_completion::register_fixture_completion; +use sqlbuild_python_project::planner_strategy::main::register_fixture_context::register_fixture_context; +use sqlbuild_python_project::planner_strategy::main::register_fixture_reads::register_fixture_reads; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!( - super::audit_entries::plan_audit_of, + crate::bindings::_helpers::planner_strategy::audit_entries::plan_audit_of, module )?)?; module.add_function(wrap_pyfunction!( - super::cursor_inputs::cursor_input_columns_of, + crate::bindings::_helpers::planner_strategy::cursor_inputs::cursor_input_columns_of, module )?)?; module.add_function(wrap_pyfunction!( - super::cursor_inputs::cursor_input_relations_of, + crate::bindings::_helpers::planner_strategy::cursor_inputs::cursor_input_relations_of, module )?)?; module.add_function(wrap_pyfunction!( - super::cursor_inputs::runtime_cursor_producer_names, + crate::bindings::_helpers::planner_strategy::cursor_inputs::runtime_cursor_producer_names, module )?)?; module.add_function(wrap_pyfunction!( - super::model_entries::resolve_microbatch_lookback, + crate::bindings::_helpers::planner_strategy::model_entries::resolve_microbatch_lookback, module )?)?; module.add_function(wrap_pyfunction!( - super::microbatch_limits::apply_microbatch_limit, + crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_microbatch_limit, module )?)?; module.add_function(wrap_pyfunction!( - super::microbatch_limits::apply_configured_microbatch_limits, + crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_configured_microbatch_limits, module )?)?; audits::register(module)?; cursor_types::register(module)?; - fixture_completion::register(module)?; - super::fixture_context::register(module)?; - super::fixture_reads::register(module)?; + register_fixture_completion(module)?; + register_fixture_context(module)?; + register_fixture_reads(module)?; microbatches::register(module)?; strategy::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs index c19156be0b..d68f6cd890 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs @@ -1,6 +1,5 @@ //! Per-model plan actions, schema actions and plan warnings. -use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyString, PyTuple}; @@ -82,7 +81,10 @@ pub(crate) fn resolve_model_plan_action<'py>( let py: Python<'py> = model.py(); let enabled: Bound<'py, PyAny> = config_value(model, "enabled")?; let strategy = ModelStrategy { - name: model_reads::model_name(model)?, + name: + sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name( + model, + )?, disabled: enabled.is_instance_of::() && !enabled.is_truthy()?, materialization: model_materialization(model)?, incremental_strategy: optional_string(&config_value(model, "incremental_strategy")?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index c51fb8fe18..877eb665ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -1,15 +1,5 @@ //! Cursor MIN/MAX inspection of selected incremental models and their watermark inputs. -use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_phases::objects::{ - by_name, config_str, model_key, native_member, optional_text, planner_error, planner_model, - text, tuple, -}; use crate::bindings::_helpers::planner_resolution::cursor_policies::resolve_start_cursor_config; use crate::bindings::_helpers::planner_resolution::cursor_windows::normalize_cursor_snapshot_grain; use crate::bindings::_helpers::planner_resolution::cursor_windows::resolve_effective_timestamp_grain; @@ -23,7 +13,6 @@ use crate::bindings::_helpers::planner_warehouse::snapshot::{ SnapshotRequest, inspection_concurrency, }; use crate::bindings::_helpers::planner_warehouse::watermarks::validate_watermark_column; -use crate::bindings::_helpers::references::source_relations::render_source_relation; use pyo3::IntoPyObject; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; @@ -50,6 +39,22 @@ use sqlbuild_python_front::cursor::main::scalars::to_inclusive::to_inclusive; use sqlbuild_python_front::cursor::main::scalars::try_parse::try_parse; use sqlbuild_python_front::model_config::main::config::config_cursor_bound::config_cursor_bound; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compile_attachment::main::resolve_cursor_input_roles::resolve_cursor_input_roles; +use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::sources::sources; +use sqlbuild_python_project::planner_phases::main::by_name::by_name; +use sqlbuild_python_project::planner_phases::main::config_str::config_str; +use sqlbuild_python_project::planner_phases::main::model_key::model_key; +use sqlbuild_python_project::planner_phases::main::native_member::native_member; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_phases::main::optional_text::optional_text; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; +use sqlbuild_python_project::planner_phases::main::planner_model::planner_model; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; +use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; use std::cell::RefCell; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -210,10 +215,13 @@ fn runtime_producer_names( return Ok(names); } let mut loaded: HashSet = HashSet::new(); - for source in project_reads::sources(project)?.try_iter()? { + for source in sources(project)?.try_iter()? { let source = source?; - if !native_field(&source_reads::source_entry(&source)?, "loader")?.is_none() { - loaded.insert(source_reads::name(&source)?.extract()?); + if !native_field(&source_entry(&source)?, "loader")?.is_none() { + loaded.insert( + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)? + .extract()?, + ); } } for key in selected.try_iter()? { @@ -239,7 +247,7 @@ fn upstream_relation<'py>( let name_text: String = name.extract()?; let is_selected = selected.is_some_and(|names| names.contains(&name_text)); let qualified = |node: Bound<'py, PyAny>| { - native_field(&resource_reads::destination(&node)?, "qualified_name") + native_field(&sqlbuild_python_project::resource_views::main::resource_reads::destination::destination(&node)?, "qualified_name") }; if (kind.eq("ref")? || kind.eq("seed")?) && !deferred.is_none() @@ -265,7 +273,7 @@ fn upstream_relation<'py>( return object( py, render_source_relation( - &source_reads::source_entry(&source)?, + &source_entry(&source)?, Some(adapter).filter(|value| !value.is_none()), )?, ); @@ -380,9 +388,9 @@ fn collect_models<'py>( }; let runtime_names = runtime_producer_names(project, &producers)?; let mut models: Vec<_> = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; - let name_value = model_reads::name_object(&model)?; + let name_value = name_object(&model)?; let model_name: String = name_value.extract()?; if !inputs.selected_keys.is_none() && !inputs.selected_keys.contains(model_key(py, &name_value)?)? @@ -422,7 +430,7 @@ fn collect_models<'py>( None => None, }; let target_info = existing_relations.get_item(&name_value)?; - let qualified = native_field(&model_reads::destination(&model)?, "qualified_name")?; + let qualified = native_field(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?, "qualified_name")?; if let Some(redirected) = redirected.filter(|value| !value.is_none()) { target_tag = Some(format!("{model_name}__target__max")); target_relation = Some(redirected.extract::()?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs index 2c063e4700..d5c1e961ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs @@ -3,7 +3,6 @@ use crate::bindings::_helpers::planner_changes::metadata::without_declared_columns_hash; use crate::bindings::_helpers::planner_migrations::fingerprint::fingerprint; use crate::bindings::_helpers::planner_migrations::fingerprint::migration_fingerprint_ref_names; -use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_warehouse::models::FingerprintWork; use pyo3::PyRefMut; use pyo3::exceptions::{PyOSError, PyValueError}; @@ -20,6 +19,7 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::planner_phases::main::object::object; use std::collections::{BTreeMap, HashMap}; use std::hash::{DefaultHasher, Hash, Hasher}; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs index 68d93023e6..d0a0ae05f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs @@ -1,8 +1,5 @@ //! Complete missing source metadata in bounded, database-grouped batches. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::planner_phases::objects::{text, tuple}; use crate::bindings::_helpers::planner_warehouse::constants::ACTIVE_CATALOG; use crate::bindings::_helpers::planner_warehouse::snapshot::{columns_for, list_relations}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -17,6 +14,11 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::references::references; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; use std::collections::BTreeSet; const SQL_ANALYSIS_KEY: &str = "sql_analysis"; @@ -194,16 +196,16 @@ pub(crate) fn semantic_source_columns<'py>( return Ok(columns.clone()); } let mut required: BTreeSet = BTreeSet::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; let analysis_off: bool = matches!( *config_node_of(&model_values(&model)?, SQL_ANALYSIS_KEY)?, ConfigNode::Bool(false) ); - if !selected_keys.contains(model_reads::model_key(&model)?)? || analysis_off { + if !selected_keys.contains(model_key(&model)?)? || analysis_off { continue; } - for reference in model_reads::references(&model)?.try_iter()? { + for reference in references(&model)?.try_iter()? { let reference = reference?; if native_field(&reference, "ref_kind")?.eq("source")? { required.insert(native_field(&reference, "ref_name")?.extract()?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs index 02cf6f7df8..a16d161f3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs @@ -1,20 +1,6 @@ //! Gather a frozen point-in-time warehouse snapshot for planning. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::fingerprints::latest_rows::select_latest_fingerprints; -use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; -use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; -use crate::bindings::_helpers::planner_identity::graph::execution_deps; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_phases::objects::{ - by_name, frozenset, model_key, native, text, tuple, -}; -use crate::bindings::_helpers::planner_scope::loaders::build_upstream_intermediate_source_map; use crate::bindings::_helpers::planner_warehouse::constants::{ ACTIVE_CATALOG, FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, MIGRATE_FROM_CONFIG_KEY, RELATIONS_MAIN_COLUMNS, RELATIONS_MAIN_LIST, @@ -43,6 +29,23 @@ use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::model_config::main::config::config_node_of::config_node_of; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::compiled_models::references::references; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::fingerprints::main::read_latest_fingerprints::read_latest_fingerprints; +use sqlbuild_python_project::fingerprints::main::select_latest_fingerprints::select_latest_fingerprints; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::graph::models::KeyCache; +use sqlbuild_python_project::planner_identity::main::execution_deps::execution_deps; +use sqlbuild_python_project::planner_phases::main::by_name::by_name; +use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; +use sqlbuild_python_project::planner_phases::main::model_key::model_key; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; +use sqlbuild_python_project::planner_scope::main::build_upstream_intermediate_source_map::build_upstream_intermediate_source_map; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; use std::collections::{BTreeSet, HashSet}; /// Everything one snapshot gather needs. @@ -82,9 +85,9 @@ pub(crate) fn destination_nodes<'py>( ) -> PyResult>> { let mut nodes: Vec<_> = Vec::new(); let collections: [Bound<'py, PyAny>; 3] = [ - project_reads::models(project)?, - project_reads::seeds(project)?, - project_reads::functions(project)?, + compiled_models(project)?, + sqlbuild_python_project::compiled_views::main::project::seeds::seeds(project)?, + sqlbuild_python_project::compiled_views::main::project::functions::functions(project)?, ]; for collection in collections { for node in collection.try_iter()? { @@ -95,7 +98,10 @@ pub(crate) fn destination_nodes<'py>( } fn selected_node(selected: &Bound<'_, PyAny>, node: &Bound<'_, PyAny>) -> PyResult { - Ok(selected.is_none() || selected.contains(resource_reads::key(node)?)?) + Ok(selected.is_none() + || selected.contains( + sqlbuild_python_project::resource_views::main::resource_reads::key::key(node)?, + )?) } fn relevant_state_keys<'py>( @@ -125,7 +131,7 @@ fn resolve_database<'py>( if !selected_node(selected, &node)? { continue; } - let database = native_field(&resource_reads::destination(&node)?, "database")?; + let database = native_field(&sqlbuild_python_project::resource_views::main::resource_reads::destination::destination(&node)?, "database")?; if !database.is_none() { return Ok(database); } @@ -142,7 +148,7 @@ fn target_schemas( if !selected_node(selected, &node)? { continue; } - let schema = native_field(&resource_reads::destination(&node)?, "schema")?; + let schema = native_field(&sqlbuild_python_project::resource_views::main::resource_reads::destination::destination(&node)?, "schema")?; if !schema.is_none() { schemas.insert(schema.extract()?); } @@ -169,7 +175,7 @@ fn model_upstream_names( selected_names: &HashSet, ) -> PyResult> { let mut names: BTreeSet = BTreeSet::new(); - for reference in model_reads::references(model)?.try_iter()? { + for reference in references(model)?.try_iter()? { let reference = reference?; let kind = native_field(&reference, "ref_kind")?; let name: String = native_field(&reference, "ref_name")?.extract()?; @@ -207,21 +213,24 @@ fn metadata_name_filter( let models = by_name(project, "models")?; let seeds = by_name(project, "seeds")?; let sources = PyDict::new(py); - for source in project_reads::sources(project)?.try_iter()? { - let entry = source_reads::source_entry(&source?)?; + for source in + sqlbuild_python_project::compiled_views::main::project::sources::sources(project)? + .try_iter()? + { + let entry = source_entry(&source?)?; sources.set_item(native_field(&entry, "name")?, entry)?; } for key in selected.try_iter()? { let name = native_field(&key?, "name")?; if let Some(model) = models.get_item(&name)? { - names.insert(native_field(&model_reads::destination(&model)?, "name")?.extract()?); + names.insert(native_field(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?, "name")?.extract()?); names.extend(model_upstream_names( &model, (&models, &seeds), &selected_names, )?); } else if let Some(seed) = seeds.get_item(&name)? { - names.insert(native_field(&seed_reads::destination(&seed)?, "name")?.extract()?); + names.insert(native_field(&sqlbuild_python_project::resource_views::main::seed_reads::destination::destination(&seed)?, "name")?.extract()?); } else if let Some(entry) = sources.get_item(&name)? { names.insert(source_table_name(&entry)?); } @@ -288,10 +297,16 @@ fn prefetch_schemas<'py>( for schema in schemas { scopes.push(PyTuple::new(py, [database.clone(), text(py, schema)])?.into_any()); } - for source in project_reads::sources(&request.project)?.try_iter()? { + for source in + sqlbuild_python_project::compiled_views::main::project::sources::sources(&request.project)? + .try_iter()? + { let source = source?; - let entry = source_reads::source_entry(&source)?; - if (!relevant.is_none() && !relevant.contains(source_reads::key(&source)?)?) + let entry = source_entry(&source)?; + if (!relevant.is_none() + && !relevant.contains( + sqlbuild_python_project::resource_views::main::source_reads::key::key(&source)?, + )?) || !native_field(&entry, "expression")?.is_none() || native_field(&entry, "schema")?.is_none() { @@ -391,8 +406,8 @@ fn gather_relations<'py>( let logical = PyDict::new(py); for node in destination_nodes(&request.project)? { logical.set_item( - location_identity(&resource_reads::destination(&node)?)?, - resource_reads::name(&node)?, + location_identity(&sqlbuild_python_project::resource_views::main::resource_reads::destination::destination(&node)?)?, + sqlbuild_python_project::resource_views::main::resource_reads::name::name(&node)?, )?; } let result = PyDict::new(py); @@ -442,7 +457,10 @@ fn gather_columns<'py>( let py = request.project.py(); let mut node_names: HashSet = HashSet::new(); for node in destination_nodes(&request.project)? { - node_names.insert(resource_reads::name(&node)?.extract()?); + node_names.insert( + sqlbuild_python_project::resource_views::main::resource_reads::name::name(&node)? + .extract()?, + ); } let mut needed: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); for (logical, relation) in relations.iter() { @@ -483,9 +501,9 @@ fn shares_unfiltered(project: &Bound<'_, PyAny>, selected: &Bound<'_, PyAny>) -> .collect::>()?; let mut all_selected = true; let mut any_migration = false; - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; - if !names.contains(&model_reads::model_name(&model)?) { + if !names.contains(&model_name(&model)?) { all_selected = false; } if config_node_of(&model_values(&model)?, MIGRATE_FROM_CONFIG_KEY)?.kind() == NodeKind::Str @@ -857,11 +875,11 @@ pub(crate) fn gather_redirected_cursor_snapshots<'py>( if let Some(model) = models.get_item(&name)? { result.set_item( name, - crate::bindings::_helpers::planner_phases::objects::replace( + sqlbuild_python_project::planner_phases::main::replace::replace( &snapshot, &[( "target_relation", - native_field(&model_reads::destination(&model)?, "qualified_name")?, + native_field(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?, "qualified_name")?, )], )?, )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs index f8eef566ff..1047e704e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs @@ -1,9 +1,6 @@ //! Warehouse source relation listing and column gathering for planning. -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::source_reads; use crate::bindings::_helpers::observations::source_capture::capture_source_columns; -use crate::bindings::_helpers::planner_phases::objects::{frozenset, text, tuple}; use crate::bindings::_helpers::planner_warehouse::constants::{ COLUMN_INFO, METADATA_NAME_FILTER_LIMIT, RELATION_LOOKUP, }; @@ -15,6 +12,11 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; +use sqlbuild_python_project::compiled_views::main::project::sources::sources; +use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; use std::collections::{BTreeSet, HashSet}; fn entries_of<'py>( @@ -23,9 +25,9 @@ fn entries_of<'py>( ) -> PyResult>> { match source_entries.filter(|entries| !entries.is_none()) { Some(entries) => entries.try_iter()?.collect(), - None => project_reads::sources(project)? + None => sources(project)? .try_iter()? - .map(|source| source_reads::source_entry(&source?)) + .map(|source| source_entry(&source?)) .collect(), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs index b93aaa112b..5b0f7ff821 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs @@ -2,15 +2,6 @@ use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::compiled_views::audit_reads; -use crate::bindings::_helpers::compiled_views::function_reads; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_phases::objects::{ - optional_text, planner_error, replace, text, -}; use crate::bindings::_helpers::planner_warehouse::constants::{ MODEL_HOOK_CONFIG_KEYS, SQL_HOOK_ENTRY, }; @@ -27,12 +18,30 @@ use sqlbuild_python_front::authored_values::main::render_project_var_text::rende use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key::model_key; +use sqlbuild_python_project::compiled_views::main::compiled_models::query_sql::query_sql; +use sqlbuild_python_project::compiled_views::main::project::audits::audits; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::functions::functions; +use sqlbuild_python_project::compiled_views::main::project::sources::sources; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_phases::main::optional_text::optional_text; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; +use sqlbuild_python_project::planner_phases::main::replace::replace; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::resource_views::main::audit_reads::references::references; +use sqlbuild_python_project::resource_views::main::audit_reads::scope_deps::scope_deps; +use sqlbuild_python_project::resource_views::main::audit_reads::sql_body::sql_body; +use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; +use sqlbuild_python_project::resource_views::main::function_reads::key::key; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::resource_views::main::source_reads::source_file::source_file; use std::collections::BTreeSet; fn managed_names(project: &Bound<'_, PyAny>) -> PyResult> { let mut names: BTreeSet<_> = BTreeSet::new(); - for source in project_reads::sources(project)?.try_iter()? { - let entry = source_reads::source_entry(&source?)?; + for source in sources(project)?.try_iter()? { + let entry = source_entry(&source?)?; if !native_field(&entry, "loader")?.is_none() { names.insert(native_field(&entry, "name")?.extract()?); } @@ -91,26 +100,23 @@ fn selected_managed_refs( managed: &BTreeSet, ) -> PyResult> { let mut names: BTreeSet<_> = BTreeSet::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model = model?; - if !selected.contains(model_reads::model_key(&model)?)? { + if !selected.contains(model_key(&model)?)? { continue; } - names.extend(sql_source_reads(&model_reads::query_sql(&model)?, managed)?); + names.extend(sql_source_reads(&query_sql(&model)?, managed)?); names.extend(hook_source_reads(&model, managed)?); } - for function in project_reads::functions(project)?.try_iter()? { + for function in functions(project)?.try_iter()? { let function = function?; - if selected.contains(function_reads::key(&function)?)? { - names.extend(sql_source_reads( - &function_reads::body_sql(&function)?, - managed, - )?); + if selected.contains(key(&function)?)? { + names.extend(sql_source_reads(&body_sql(&function)?, managed)?); } } - for audit in project_reads::audits(project)?.try_iter()? { + for audit in audits(project)?.try_iter()? { let audit = audit?; - let deps = audit_reads::scope_deps(&audit)?; + let deps = scope_deps(&audit)?; if deps.is_truthy()? { let mut any = false; for dep in deps.try_iter()? { @@ -123,14 +129,14 @@ fn selected_managed_refs( continue; } } - for reference in audit_reads::references(&audit)?.try_iter()? { + for reference in references(&audit)?.try_iter()? { let reference = reference?; let name: String = native_field(&reference, "ref_name")?.extract()?; if native_field(&reference, "ref_kind")?.eq("source")? && managed.contains(&name) { names.insert(name); } } - names.extend(sql_source_reads(&audit_reads::sql_body(&audit)?, managed)?); + names.extend(sql_source_reads(&sql_body(&audit)?, managed)?); } Ok(names) } @@ -173,9 +179,9 @@ fn source_target_name<'py>( } fn raw_source_entry<'py>(source: &Bound<'py, PyAny>) -> PyResult> { - let declared = source_reads::source_entry(source)?; + let declared = source_entry(source)?; let name = native_field(&declared, "name")?; - for entry in native_field(&source_reads::source_file(source)?, "source_entries")?.try_iter()? { + for entry in native_field(&source_file(source)?, "source_entries")?.try_iter()? { let entry = entry?; if native_field(&entry, "name")?.eq(&name)? { return Ok(entry); @@ -306,10 +312,9 @@ pub(crate) fn build_source_read_map<'py>( let target = resolve_target(project_config, local_config, target_name)?; let effective_vars = native_field(project, "effective_vars")?; let result = source_map.copy()?; - for source in project_reads::sources(project)?.try_iter()? { + for source in sources(project)?.try_iter()? { let source = source?; - let name: String = - native_field(&source_reads::source_entry(&source)?, "name")?.extract()?; + let name: String = native_field(&source_entry(&source)?, "name")?.extract()?; if !selected.contains(&name) { continue; } @@ -339,8 +344,8 @@ pub(crate) fn with_declared_source_reads<'py>( return Ok(source_map.clone()); } let result = source_map.copy()?; - for source in project_reads::sources(project)?.try_iter()? { - let entry = source_reads::source_entry(&source?)?; + for source in sources(project)?.try_iter()? { + let entry = source_entry(&source?)?; let name: String = native_field(&entry, "name")?.extract()?; if missing.contains(&name) { result.set_item(name, entry)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs index 61c4672fa8..5c8143ab3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs @@ -1,13 +1,9 @@ //! Direct planner source freshness observation and the warnings for unknown freshness. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::resource_reads; use crate::bindings::_helpers::freshness::planning::PREVIEW_MESSAGE; use crate::bindings::_helpers::freshness::planning::{ build_direct_source_freshness_planning_result, build_direct_source_freshness_propagation_result, }; -use crate::bindings::_helpers::planner_phases::objects::{native, replace, text, tuple}; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::constants::RUN_SCOPE; use crate::bindings::_helpers::planner_warehouse::snapshot::{destination_nodes, lower}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -25,6 +21,11 @@ use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; +use sqlbuild_python_project::planner_phases::main::native::native; +use sqlbuild_python_project::planner_phases::main::replace::replace; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::BTreeSet; /// One warning per reason that left configured source freshness unknown. @@ -72,7 +73,7 @@ pub(crate) fn source_freshness_unknown_warnings<'py>( fn state_database<'py>(project: &Bound<'py, PyAny>) -> PyResult> { for node in destination_nodes(project)? { - let database = native_field(&resource_reads::destination(&node)?, "database")?; + let database = native_field(&sqlbuild_python_project::resource_views::main::resource_reads::destination::destination(&node)?, "database")?; if !database.is_none() { return Ok(database); } @@ -86,7 +87,7 @@ fn state_schemas( ) -> PyResult> { let mut schemas: BTreeSet<_> = BTreeSet::new(); for node in destination_nodes(project)? { - let schema = native_field(&resource_reads::destination(&node)?, "schema")?; + let schema = native_field(&sqlbuild_python_project::resource_views::main::resource_reads::destination::destination(&node)?, "schema")?; if !schema.is_none() { schemas.insert(schema.extract()?); } @@ -100,7 +101,7 @@ fn state_schemas( let Some(model) = data.model(scope.py(), name) else { continue; }; - let schema = native_field(&model_reads::destination(&model)?, "schema")?; + let schema = native_field(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?, "schema")?; if !schema.is_none() { schemas.insert(schema.extract()?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs index 96362f5a61..ed0277c229 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs @@ -1,13 +1,8 @@ //! Selected-scope validation that external source tables exist in the warehouse. -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::planner_phases::objects::{planner_error, tuple}; -use crate::bindings::_helpers::planner_scope::retained::scope_data; use crate::bindings::_helpers::planner_warehouse::constants::{ MISSING_SOURCE_TABLE_CODE, RUN_BOUNDED, WILDCARD_TABLE_CHARACTER, }; -use crate::bindings::_helpers::references::source_relations::render_source_relation; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; @@ -27,6 +22,12 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_front::discovery::main::project::yaml_entry_line::yaml_entry_line; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; +use sqlbuild_python_project::planner_phases::main::tuple::tuple; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; +use sqlbuild_python_project::resource_views::main::source_reads::source_file::source_file; use std::collections::HashMap; fn selected_source_readers(scope: &Bound<'_, PyAny>) -> PyResult>> { @@ -80,13 +81,14 @@ fn read_relation<'py>( } fn declaration_location(source: &Bound<'_, PyAny>) -> PyResult { - let file = source_reads::source_file(source)?; + let file = source_file(source)?; let path: String = native_field(&file, "relative_path")? .call_method0("as_posix")? .extract()?; let line = yaml_entry_line( &native_field(&file, "contents")?.extract::()?, - &source_reads::name(source)?.extract::()?, + &sqlbuild_python_project::resource_views::main::source_reads::name::name(source)? + .extract::()?, ); Ok(match line { None => path, @@ -106,13 +108,17 @@ pub(crate) fn check_selected_source_tables_exist<'py>( let read_map = native_field(relations, "source_read_map")?; let listed = native_field(relations, "listed_source_names")?; let mut unlisted: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>, Vec)> = Vec::new(); - for source in project_reads::sources(project)?.try_iter()? { + for source in sqlbuild_python_project::compiled_views::main::project::sources::sources(project)? + .try_iter()? + { let source = source?; - let name: String = source_reads::name(&source)?.extract()?; + let name: String = + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)? + .extract()?; let Some(source_readers) = readers.get(&name).filter(|readers| !readers.is_empty()) else { continue; }; - let declared = source_reads::source_entry(&source)?; + let declared = source_entry(&source)?; let entry = mapping_get_or(&read_map, &name, &declared)?; if !external_table(&declared)? || !external_table(&entry)? { continue; @@ -162,7 +168,10 @@ pub(crate) fn check_selected_source_tables_exist<'py>( } let role = native_field(&probe, "role")?; missing.push(MissingSourceTable { - source_name: source_reads::name(&source)?.extract()?, + source_name: sqlbuild_python_project::resource_views::main::source_reads::name::name( + &source, + )? + .extract()?, relation: String::new(), read_by, unreadable: status.eq("missing_or_unreadable")?, @@ -183,9 +192,14 @@ pub(crate) fn check_selected_source_tables_exist<'py>( } let message = missing_source_tables_message(&missing); let sources = PyDict::new(py); - for source in project_reads::sources(project)?.try_iter()? { + for source in sqlbuild_python_project::compiled_views::main::project::sources::sources(project)? + .try_iter()? + { let source = source?; - sources.set_item(source_reads::name(&source)?, source)?; + sources.set_item( + sqlbuild_python_project::resource_views::main::source_reads::name::name(&source)?, + source, + )?; } for item in missing.iter_mut().take(10) { if let Some(source) = sources.get_item(&item.source_name)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs index b1adf98e3f..ea9d541ea8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs @@ -1,10 +1,5 @@ //! Validate reliable watermark column contracts before warehouse inspection. -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; -use crate::bindings::_helpers::compiled_views::source_reads; -use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_phases::objects::{config_str, planner_error, text}; use crate::bindings::_helpers::planner_strategy::cursor_types::classify_cursor_sql_type; use crate::bindings::_helpers::planner_warehouse::constants::{ CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, WATERMARK_CONTRACT_CODE, @@ -15,6 +10,12 @@ use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::model_config::main::config::config_value::config_value; use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::planner_phases::main::config_str::config_str; +use sqlbuild_python_project::planner_phases::main::object::object; +use sqlbuild_python_project::planner_phases::main::planner_error::planner_error; +use sqlbuild_python_project::planner_phases::main::text::text; +use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::schema_entry; +use sqlbuild_python_project::resource_views::main::source_reads::source_entry::source_entry; use std::collections::HashMap; const CONTRACT_KEY: &str = "contract"; @@ -75,7 +76,10 @@ pub(crate) fn validate_watermark_column<'py>( let py = model.py(); let kind = native_field(reference, "ref_kind")?; let ref_name: String = native_field(reference, "ref_name")?.extract()?; - let model_name: String = model_reads::model_name(model)?; + let model_name: String = + sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name( + model, + )?; let lowered = |value: &str| -> PyResult { text(py, value).call_method0("lower")?.extract() }; let (names, types) = if kind.eq("ref")? { @@ -112,7 +116,7 @@ pub(crate) fn validate_watermark_column<'py>( let Some(source) = sources.get_item(&ref_name)? else { return Ok(()); }; - let entry = source_reads::source_entry(&source)?; + let entry = source_entry(&source)?; if !native_field(&entry, "contract")?.eq("enforced")? { return Ok(()); } @@ -121,7 +125,7 @@ pub(crate) fn validate_watermark_column<'py>( let Some(seed) = seeds.get_item(&ref_name)? else { return Ok(()); }; - declared_columns(&native_field(&seed_reads::schema_entry(&seed)?, "columns")?)? + declared_columns(&native_field(&schema_entry(&seed)?, "columns")?)? } else { return Ok(()); }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index e21f9cb61c..57da7f3ed6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -8,17 +8,15 @@ use pyo3::types::{PyDict, PyDictMethods, PyString}; use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; -use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - attached_columns, attached_schema, -}; -use crate::bindings::_helpers::compiled_views::source_views::{ - AttachedSource, SourceViews, read_source_view, -}; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::enum_member_value::enum_member_value; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_project::main::python_column_facts::python_column_facts; +use sqlbuild_python_project::compiled_views::main::model_inputs::attached_columns::attached_columns; +use sqlbuild_python_project::compiled_views::main::model_inputs::attached_schema::attached_schema; +use sqlbuild_python_project::resource_views::main::source_views::read_source_view::read_source_view; +use sqlbuild_python_project::resource_views::types::{AttachedSource, SourceViews}; const UNKNOWN_TYPE: &str = "UNKNOWN"; const NOT_NULL_AUDIT_NAME: &str = "not_null"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs index e72461f44f..4312270b18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -17,7 +17,6 @@ use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_model_config::templates::models::Scalar; use sqlbuild_python_base::positional_record; -use crate::bindings::_helpers::compiled_views::model_input_view::input_reference_rows; use crate::bindings::_helpers::project_assembly::models::{ NativeProjectResources, NativeResourceFailure, }; @@ -34,6 +33,7 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_front::authored_values::main::project_var_text::project_var_text; use sqlbuild_python_front::model_config::main::config::record_reads::record_reads; +use sqlbuild_python_project::compiled_views::main::model_inputs::input_reference_rows::input_reference_rows; const OS_MODULE: &str = "os"; const PROJECT_ASSEMBLY_CONTEXT: &str = "native project assembly"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs index 64cb409ce9..e05f8753e0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs @@ -6,10 +6,10 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::header_metadata::models::DynamicFamily; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; -use crate::bindings::_helpers::compiled_views::model_input_view::attached_schema; use crate::bindings::_helpers::project_assembly::syntax_validation::hook_sql_statements; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::attachments::main::analysis_cursor_sql::analysis_cursor_sql; +use sqlbuild_python_project::compiled_views::main::model_inputs::attached_schema::attached_schema; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs index be9ac7a81a..866850a577 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs @@ -9,13 +9,13 @@ use sqlbuild_core::query_command::main::render_dag_summary::render_dag_summary; use crate::bindings::_helpers::compile_driver::project::build_project_graph; use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; fn array_length(document: &serde_json::Value, key: &str) -> PyResult { document diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index bfab8b233a..e87eda9f4e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -36,7 +36,6 @@ use sqlbuild_recall::writer::models::{ DeliverySummary, RecallSession, SessionConfig, SessionStore, SetupOutcome, }; -use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::output_capture::records::invocation_context; use crate::bindings::_helpers::recall::clickhouse_store::{ ClickHouseLocation, ClickHouseStoreOpener, missing_database_reason, @@ -53,6 +52,7 @@ use crate::bindings::_helpers::recall::replacements::{RecallReplacements, in_rep use crate::bindings::_helpers::recall::sinks::InvocationEvents; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; /// What the invocation records: its session, when enabled, and the resolved target and adapter. #[derive(Clone, Debug, Default)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs index a9c5c43c85..e238e4e34c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs @@ -4,13 +4,13 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; -use crate::bindings::_helpers::compiled_views::relation_keys::entry_relation_key_row; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::entry_relation_key_row::entry_relation_key_row; /// `CompilerDiagnostic` positions. const DIAGNOSTIC_PHASE_FIELD: usize = 0; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index ea894c5e18..42514a58ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -1,9 +1,6 @@ //! Native behaviour of the hosted shapes: computed attributes, methods, ordering and validation. use crate::bindings::_helpers::freshness::observation::normalize_observation; -use crate::bindings::_helpers::lineage::lineage_facts::{ - facts_eq, facts_getitem, facts_iter, facts_len, facts_projected, facts_resource_name, -}; use crate::bindings::_helpers::record_behaviours::auditing::{ finish_measurement_contract, finish_threshold_bound, finish_thresholds, }; @@ -73,14 +70,20 @@ use crate::bindings::_helpers::record_behaviours::values::{ finish_fingerprint_set, logical_type_display_name, sql_value_kind, }; use crate::bindings::_helpers::{ - analysis, analysis_session, build_progress, compile_command, compiled_views, compute_logs, - dbt_events, dbt_interop, event_exporting, execution_history, graph, planner_entries, - planner_migrations, planner_phases, project_assembly, result_output, rules, + analysis, analysis_session, build_progress, compile_command, compute_logs, dbt_events, + dbt_interop, event_exporting, execution_history, planner_entries, planner_migrations, + planner_phases, project_assembly, result_output, rules, }; use sqlbuild_python_base::records::models::{GetterEntry, RecordBehaviours}; use sqlbuild_python_base::records::types::ClassMethod; use sqlbuild_python_base::records::types::ClassProperty; use sqlbuild_python_base::records::types::RecordFinish; +use sqlbuild_python_project::lineage::main::facts_eq::facts_eq; +use sqlbuild_python_project::lineage::main::facts_getitem::facts_getitem; +use sqlbuild_python_project::lineage::main::facts_iter::facts_iter; +use sqlbuild_python_project::lineage::main::facts_len::facts_len; +use sqlbuild_python_project::lineage::main::facts_projected::facts_projected; +use sqlbuild_python_project::lineage::main::facts_resource_name::facts_resource_name; /// Compiler records whose bound values are normalized or validated natively. pub(crate) const BEHAVIOUR_FINISHERS: &[(&str, RecordFinish)] = &[ @@ -359,7 +362,7 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ build_progress::getters::GETTERS, compile_command::getters::GETTERS, sqlbuild_python_base::compile_context::constants::GETTERS, - compiled_views::getters::GETTERS, + sqlbuild_python_project::compiled_views::constants::GETTERS, compute_logs::getters::GETTERS, dbt_events::getters::GETTERS, dbt_interop::getters::GETTERS, @@ -367,7 +370,7 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ sqlbuild_python_front::discovery::constants::GETTERS, event_exporting::getters::GETTERS, execution_history::getters::GETTERS, - graph::getters::GETTERS, + sqlbuild_python_project::graph::constants::GETTERS, sqlbuild_python_front::model_config::constants::GETTERS, planner_entries::getters::GETTERS, planner_migrations::getters::GETTERS, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs index f93c691b4e..a7992a4914 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs @@ -4,12 +4,13 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyTuple, PyTupleMethods}; -use crate::bindings::_helpers::graph::project_graph::{ResourceRow, project_graph}; use crate::bindings::_helpers::record_behaviours::compiled::compile_input_error; use sqlbuild_python_base::boundary::main::call_argument::call_argument; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::render::main::declarations::constant_declaration_sql::constant_declaration_sql; use sqlbuild_python_front::render::main::declarations::render_enum_member_value::render_enum_member_value; +use sqlbuild_python_project::graph::main::project::project_graph::project_graph; +use sqlbuild_python_project::graph::models::ResourceRow; /// The parameter name through which a macro receives its context. const MACRO_CONTEXT_PARAMETER_NAME: &str = "ctx"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/python_nodes.rs index 3b5ab27c60..6ab2e79409 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/python_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/python_nodes.rs @@ -6,10 +6,10 @@ use pyo3::types::{ }; use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; -use crate::bindings::_helpers::graph::node_set::PyNodeSet; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; +use sqlbuild_python_project::graph::models::PyNodeSet; /// `PythonNodeGraph.selected_sql_refs(selected_names=...)`: SQL references of the selected closure. pub(crate) fn graph_selected_sql_refs( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs index f183f2d284..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs @@ -1,5 +1 @@ -pub(crate) mod explicit_references; -pub(crate) mod reference_facts; pub(crate) mod registration; -pub(crate) mod relations; -pub(crate) mod source_relations; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/references/registration.rs index cacffa8feb..53bb674dcf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/references/registration.rs @@ -1,6 +1,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; +use sqlbuild_python_project::references::main::register_relations::register_relations; +use sqlbuild_python_project::references::main::register_source_relations::register_source_relations; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - crate::bindings::_helpers::references::relations::register(module)?; - crate::bindings::_helpers::references::source_relations::register(module) + register_relations(module)?; + register_source_relations(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/source_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/references/source_relations.rs deleted file mode 100644 index 37d3d0d621..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/source_relations.rs +++ /dev/null @@ -1,137 +0,0 @@ -//! A declared source rendered as the SQL table factor that reads it. - -use sqlbuild_host::entries::models::AdapterHost; - -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyDict, PyDictMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -/// The SQL analysis dialect whose engines read replacing tables with `FINAL`. -const FINAL_READ_DIALECT: &str = "clickhouse"; -/// Error code for a `final` source on another adapter. -const FINAL_UNSUPPORTED_CODE: &str = "A010"; -/// Guidance for a `final` source on another adapter. -const FINAL_UNSUPPORTED_HELP: &str = - "Remove 'final' from the source; only ClickHouse reads sources with FINAL."; - -/// Leading words of a source expression that is a query rather than a relation. -const QUERY_PREFIXES: [&str; 3] = ["select", "with", "values"]; - -/// A source expression as a table factor: queries parenthesized, relations as written. -fn expression_relation(expression: &str) -> String { - let trimmed: &str = expression.trim(); - let stripped: &str = trimmed.strip_suffix(';').unwrap_or(trimmed).trim(); - if stripped.starts_with('(') { - return stripped.to_owned(); - } - let lowered: String = stripped.to_lowercase(); - if QUERY_PREFIXES - .iter() - .any(|prefix| lowered.starts_with(prefix)) - { - return format!("({stripped})"); - } - stripped.to_owned() -} - -/// Render a source as a SQL table factor, through the adapter when one is given. -#[pyfunction] -#[pyo3(signature = (*, entry, adapter = None))] -pub(crate) fn render_source_relation<'py>( - entry: &Bound<'py, PyAny>, - adapter: Option<&Bound<'py, PyAny>>, -) -> PyResult { - let adapter: Option<&Bound<'py, PyAny>> = adapter.filter(|adapter| !adapter.is_none()); - let expression: Option = native_field(entry, "expression")?.extract()?; - if let Some(expression) = expression { - return match adapter { - Some(adapter) => { - let keywords: Bound<'py, PyDict> = PyDict::new(entry.py()); - keywords.set_item("expression", expression)?; - adapter - .call_method("render_source_expression_relation", (), Some(&keywords))? - .extract() - } - None => Ok(expression_relation(&expression)), - }; - } - let table: Option = native_field(entry, "table")?.extract()?; - let table: String = match table { - Some(table) => table, - None => native_field(entry, "name")?.extract()?, - }; - let database: Option = native_field(entry, "database")?.extract()?; - let schema: Option = native_field(entry, "schema")?.extract()?; - if let Some(adapter) = adapter { - let keywords: Bound<'py, PyDict> = PyDict::new(entry.py()); - keywords.set_item("database", &database)?; - keywords.set_item("schema", &schema)?; - keywords.set_item("name", &table)?; - let rendered: Option = adapter - .call_method("render_qualified_name", (), Some(&keywords))? - .extract()?; - if let Some(rendered) = rendered { - return Ok(rendered); - } - } - Ok([database, schema, Some(table)] - .into_iter() - .flatten() - .collect::>() - .join(".")) -} - -/// A source as the table factor that reads it, deduplicated with `FINAL` when `final`. -pub(crate) fn render_source_read<'py>( - entry: &Bound<'py, PyAny>, - adapter: &Bound<'py, PyAny>, -) -> PyResult { - let relation: String = render_source_relation(entry, Some(adapter))?; - if !native_field(entry, "final")?.is_truthy()? { - return Ok(relation); - } - let dialect: Option = native_field(adapter, "sql_analysis_dialect_name")?.extract()?; - if dialect.as_deref() != Some(FINAL_READ_DIALECT) { - return Err(final_unsupported( - entry.py(), - &native_field(adapter, "adapter_name")?.str()?.to_string(), - )); - } - Ok(format!("(SELECT * FROM {relation} FINAL)")) -} - -/// The coded error for a `final` source on an adapter that cannot read with `FINAL`. -fn final_unsupported(py: Python<'_>, adapter_name: &str) -> PyErr { - match host_entry(py, AdapterHost::AdapterUserError).and_then(|class| { - call_with( - &class, - &[ - ( - "message", - PyString::new( - py, - &format!( - "adapter '{adapter_name}' does not support the source option 'final'" - ), - ) - .into_any(), - ), - ("code", PyString::new(py, FINAL_UNSUPPORTED_CODE).into_any()), - ("help", PyString::new(py, FINAL_UNSUPPORTED_HELP).into_any()), - ], - ) - }) { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(render_source_relation, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs deleted file mode 100644 index 97f7cf9351..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs +++ /dev/null @@ -1,60 +0,0 @@ -//! Generated relation names fitted within identifier limits, raising `SharedInputError`. - -use pyo3::prelude::{Bound, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::identifier_fitting::main::fit_artifact_logical_name::fit_artifact_logical_name as fit_logical; -use sqlbuild_analysis::identifier_fitting::main::fit_auxiliary_relation_name::fit_auxiliary_relation_name as fit_auxiliary; -use sqlbuild_host::entries::models::FacadeHost; -use sqlbuild_python_base::native_host::main::host_error::host_error; - -use sqlbuild_python_base::boundary::models::Request; - -pub(crate) fn shared_error(py: Python<'_>, message: String) -> PyErr { - host_error(py, FacadeHost::SharedInputError, &message) -} - -/// A readable logical component fitted after a fixed prefix, with a hash suffix when shortened. -#[pyfunction] -#[pyo3(signature = (**options))] -fn fit_artifact_logical_name(options: Option<&Bound<'_, PyDict>>) -> PyResult { - let request = Request::new("fit_artifact_logical_name", options)?; - fitted_logical_name( - request.py(), - ( - &request.get("logical_name")?.extract::()?, - &request.get("fixed_prefix")?.extract::()?, - ), - request.get("identifier_limit")?.extract()?, - &request.get("artifact_label")?.extract::()?, - ) -} - -/// `fit_artifact_logical_name`, raising `SharedInputError` when the name cannot fit. -pub(crate) fn fitted_logical_name( - py: Python<'_>, - names: (&str, &str), - identifier_limit: i64, - artifact_label: &str, -) -> PyResult { - fit_logical(names, identifier_limit, artifact_label) - .map_err(|message| shared_error(py, message)) -} - -/// `base_name + suffix` fitted within the limit, keeping the suffix and a base hash. -#[pyfunction] -#[pyo3(signature = (*, base_name, suffix, identifier_limit))] -fn fit_auxiliary_relation_name( - py: Python<'_>, - base_name: &str, - suffix: &str, - identifier_limit: i64, -) -> PyResult { - fit_auxiliary(base_name, suffix, identifier_limit).map_err(|message| shared_error(py, message)) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(fit_artifact_logical_name, module)?)?; - module.add_function(wrap_pyfunction!(fit_auxiliary_relation_name, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/mod.rs index 0bbf05581d..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/mod.rs @@ -1,2 +1 @@ -pub(crate) mod fitting; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/registration.rs index 7a489501cf..240f77f73a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/registration.rs @@ -2,8 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::relation_naming::fitting; +use sqlbuild_python_project::relation_naming::main::register_fitting::register_fitting; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - fitting::register(module) + register_fitting(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs index 8af26d5baa..4ba0c73a2b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs @@ -18,7 +18,6 @@ use sqlbuild_cli::diff_report::types::DiffNamedValue; use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::compile_driver::diff::run_diff_pipeline; -use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::dagster_invocation::access::string_tuple as texts; use crate::bindings::_helpers::diff_report::command::target_max_full_rows; use crate::bindings::_helpers::diff_report::facts::diff_facts; @@ -33,6 +32,7 @@ use sqlbuild_python_base::build_command::main::boolean::boolean; use sqlbuild_python_base::build_command::main::text::text; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; const KEYED_CHECK: &str = "keyed comparison"; const MAX_EXAMPLE_LENGTH: usize = 80; @@ -78,7 +78,7 @@ fn models_named<'py>( names: &[String], ) -> PyResult)>> { let mut models: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model: Bound<'py, PyAny> = model?; let name: String = native_field(&model, "name")?.extract()?; if names.contains(&name) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs index 85d1a7c0ba..e87d6ba7e5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/impact.rs @@ -7,12 +7,14 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyFrozenSet, PyString}; use sqlbuild_analysis::review_command::models::NodeComparison; -use crate::bindings::_helpers::compiled_views::{model_reads, project_reads}; use crate::bindings::_helpers::lineage::column_lineage_graph::ProjectColumnLineage; use crate::bindings::_helpers::lineage::project_lineage::build_project_column_lineage; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_names::inferred_names; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; /// The columns of a changed model a comparison proved changed, or `None` when every column may. fn changed_columns(comparison: Option<&NodeComparison>) -> Option> { @@ -44,9 +46,9 @@ pub(crate) fn affected_columns( ) -> PyResult>> { let py: Python<'_> = project.py(); let mut models: BTreeMap> = BTreeMap::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model: Bound<'_, PyAny> = model?; - let name: String = model_reads::model_name(&model)?; + let name: String = model_name(&model)?; if affected.contains(&name) { models.insert(name, model); } @@ -72,7 +74,7 @@ pub(crate) fn affected_columns( }; let columns: Vec = match changed_columns(comparisons.get(name)) { Some(columns) => columns, - None => model_reads::inferred_names(model)?.unwrap_or_default(), + None => inferred_names(model)?.unwrap_or_default(), }; pending.extend(columns.into_iter().map(|column| (name.clone(), column))); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs index 0a225d96f2..0f2f9634b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs @@ -24,12 +24,7 @@ use crate::bindings::_helpers::change_selection::definition_snapshot::project_wi use crate::bindings::_helpers::change_selection::selection::{ChangedProject, changed_project}; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::project::build_project_graph; -use crate::bindings::_helpers::compiled_views::model_reads; -use crate::bindings::_helpers::compiled_views::project_reads; -use crate::bindings::_helpers::compiled_views::seed_reads; use crate::bindings::_helpers::execution_history::conversion::optional_text; -use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; -use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; use crate::bindings::_helpers::review_command::support::review_error; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::build_command::main::boolean::boolean; @@ -41,6 +36,12 @@ use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::reso use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_front::discovery::main::project::configured_git_base_ref::configured_git_base_ref; use sqlbuild_python_front::discovery::main::project::configured_review::configured_review; +use sqlbuild_python_project::compiled_views::main::compiled_models::dep_rows::dep_rows; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; +use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; +use sqlbuild_python_project::compiled_views::main::project::seeds::seeds; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; +use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; const BASE_REF_ALIAS: &str = "@base"; const UDF_KIND: &str = "udf"; @@ -213,12 +214,15 @@ fn key_columns(value: &Bound<'_, PyAny>) -> PyResult> { /// One model as a review node, critical by its `critical` key or one of the critical tags. fn model_node(model: &Bound<'_, PyAny>, critical_tags: &[String]) -> PyResult { - let values: Bound<'_, PyAny> = native_field(&model_reads::config(model)?, "values")?; + let values: Bound<'_, PyAny> = native_field( + &sqlbuild_python_project::compiled_views::main::compiled_models::config::config(model)?, + "values", + )?; let tags: Vec = strings(&config_value(&values, "tags")?)?; let marked: Bound<'_, PyAny> = config_value(&values, "critical")?; let materialized: Bound<'_, PyAny> = config_value(&values, "materialized")?; Ok(ReviewNode { - name: model_reads::model_name(model)?, + name: model_name(model)?, kind: MODEL_KIND.to_owned(), critical: (!marked.is_none() && marked.is_truthy()?) || tags.iter().any(|tag| critical_tags.contains(tag)), @@ -377,10 +381,16 @@ fn review_graph( stateful: BTreeSet::new(), databases: BTreeMap::new(), }; - for seed in project_reads::seeds(project)?.try_iter()? { + for seed in seeds(project)?.try_iter()? { let seed: Bound<'_, PyAny> = seed?; - let name: String = seed_reads::name(&seed)?.extract()?; - if let Some(database) = destination_database(&seed_reads::destination(&seed)?)? { + let name: String = + sqlbuild_python_project::resource_views::main::seed_reads::name::name(&seed)? + .extract()?; + if let Some(database) = destination_database( + &sqlbuild_python_project::resource_views::main::seed_reads::destination::destination( + &seed, + )?, + )? { project_graph.databases.insert(name.clone(), database); } project_graph @@ -397,7 +407,7 @@ fn review_graph( ); } let mut dependencies: Vec<(String, Vec<(String, String)>)> = Vec::new(); - for model in project_reads::models(project)?.try_iter()? { + for model in compiled_models(project)?.try_iter()? { let model: Bound<'_, PyAny> = model?; let node: ReviewNode = model_node(&model, critical_tags)?; if node.materialized == INCREMENTAL_MATERIALIZATION @@ -405,13 +415,13 @@ fn review_graph( { project_graph.stateful.insert(node.name.clone()); } - if let Some(database) = destination_database(&model_reads::destination(&model)?)? { + if let Some(database) = destination_database(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?)? { project_graph.databases.insert(node.name.clone(), database); } project_graph .keys .insert(node.name.clone(), node_key(MODEL_KIND, &node.name)); - dependencies.push((node.name.clone(), model_reads::dep_rows(&model)?)); + dependencies.push((node.name.clone(), dep_rows(&model)?)); project_graph.graph.nodes.insert(node.name.clone(), node); } let mut sources: BTreeMap> = BTreeMap::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs index c0cb55ff21..9fe21fe6b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs @@ -13,7 +13,6 @@ use sqlbuild_rules::models::RulesConfig; use sqlbuild_rules::rule_catalogue::main::sorted_catalogue_order::sorted_catalogue_order; use sqlbuild_rules::rule_catalogue::models::{CatalogueRule, OptionKind, OptionSpec, OptionValue}; -use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; use crate::bindings::_helpers::rules::builtin_catalogue::builtin_rules; use crate::bindings::_helpers::rules::config_json::asdict_json; use crate::bindings::_helpers::rules::custom_rule_evidence::verify_custom_rules as verify_rule_sources; @@ -27,6 +26,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as record_field; use sqlbuild_python_base::records::main::fields::path_posix::path_posix; use sqlbuild_python_base::records::main::fields::replaced::replaced; +use sqlbuild_python_project::planner_scope::main::build_planner_scope::build_planner_scope; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs index 93ca3aed85..372d122c2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_host_payload.rs @@ -12,17 +12,18 @@ use pyo3::types::{ use serde_json::{Map, Value, json}; use sqlbuild_discovery::models::LineColumnSpan; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::model_reads::enum_columns; -use crate::bindings::_helpers::compiled_views::records::{ - compiled_audit_record, compiled_function_record, compiled_seed_record, compiled_source_record, -}; use crate::bindings::_helpers::rules::custom_host_columns::PayloadColumn; use crate::bindings::_helpers::rules::custom_hosts::pickled; use crate::bindings::_helpers::rules::fact_replay::{ ModelEntry, project_attribute, project_models, }; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::compiled_views::main::compiled_models::enum_columns::enum_columns; +use sqlbuild_python_project::compiled_views::main::project::compiled_audit_record::compiled_audit_record; +use sqlbuild_python_project::compiled_views::main::project::compiled_function_record::compiled_function_record; +use sqlbuild_python_project::compiled_views::main::project::compiled_seed_record::compiled_seed_record; +use sqlbuild_python_project::compiled_views::main::project::compiled_source_record::compiled_source_record; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; /// The first bytes of every payload file; hosts reject any other. pub(crate) const CUSTOM_HOST_PAYLOAD_MAGIC: &[u8; 8] = b"SQBHOST1"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs index f51c455187..7a5a3a8d2f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs @@ -21,11 +21,11 @@ use sqlbuild_rules::custom_rule_cache::models::{ }; use sqlbuild_rules::custom_rule_cache::types::{ReadSet, StoredRules, SubjectResults}; -use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::rules::findings::{FindingClasses, invalid_finding}; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_project::compiled_project::main::plain_value::plain_value; /// One custom rule's cache identity from its metadata, source fingerprint and settings. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs index 6e9e3d08a3..4fbdffac47 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs @@ -17,10 +17,6 @@ use sqlbuild_rules::engine::main::start_speculative_rows::start_speculative_rows use sqlbuild_rules::errors::RowsError; use sqlbuild_rules::models::{RowsEvaluation, RowsRequest, SpeculativeAnswer, SpeculativeRows}; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_views::records::{ - compiled_project_facts, record_model_analyses, -}; use crate::bindings::_helpers::rules::catalogue::references_custom_rules; use crate::bindings::_helpers::rules::rule_dispatch::speculative_rules_request; use crate::bindings::_helpers::rules::rule_plan::{rule_run_plan, selects_row_rules}; @@ -28,6 +24,9 @@ use crate::bindings::_helpers::rules::rules_config::load_rules_config_with_cache use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_views::main::project::compiled_project_facts::compiled_project_facts; +use sqlbuild_python_project::compiled_views::main::project::record_model_analyses::record_model_analyses; const CANCELLED: &str = "native rules evaluation was cancelled"; const BUILD_PHASE: &str = "early_rules_build_ms"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index 2b8ca2857c..efb853e5fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -22,10 +22,6 @@ use sqlbuild_rules::rules::main::selected_sql_codes::selected_sql_codes; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::ExpansionVars; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::present; -use crate::bindings::_helpers::compiled_views::relation_keys::model_key_row; -use crate::bindings::_helpers::compiled_views::relation_keys::{ - RelationKeyRow, RelationKeys, one_row_seed_names, relation_keys, source_key_input, -}; use crate::bindings::_helpers::rules::lint_expansion::{ LintExpansionSources, lint_expansion_inputs, }; @@ -40,6 +36,12 @@ use sqlbuild_python_base::native_threads::main::join_interruptibly::join_interru use sqlbuild_python_base::native_threads::models::Joined; use sqlbuild_python_base::records::main::fields::field::field as record_field; use sqlbuild_python_front::render::models::SqlExpansionInputs; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_key_row::model_key_row; +use sqlbuild_python_project::compiled_views::main::compiled_models::one_row_seed_names::one_row_seed_names; +use sqlbuild_python_project::compiled_views::main::compiled_models::relation_keys::relation_keys; +use sqlbuild_python_project::compiled_views::main::compiled_models::source_key_input::source_key_input; +use sqlbuild_python_project::compiled_views::models::RelationKeyRow; +use sqlbuild_python_project::compiled_views::types::RelationKeys; /// The lint thresholds used when the project leaves them unset. const DEFAULT_MAX_LITERAL_LENGTH: i64 = 100; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs index 564ae8ea7d..948466a24e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs @@ -13,12 +13,6 @@ use sqlbuild_rules::project_tree::main::resolve_project_path::resolve_project_pa use sqlbuild_rules::project_tree::main::tracked_text_path::tracked_text_path; use sqlbuild_rules::project_tree::models::ProjectTreeEntry; -use crate::bindings::_helpers::compiled_views::audit_views::NativeCompiledAudit; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::function_views::NativeCompiledFunction; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; -use crate::bindings::_helpers::compiled_views::seed_views::NativeCompiledSeed; -use crate::bindings::_helpers::compiled_views::source_views::NativeCompiledSource; use crate::bindings::_helpers::rules::fact_encoder::{ Encoded, FactEncoder, FactFailure, error_digest, value_digest, }; @@ -31,6 +25,12 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as record_field_or_attribute; use sqlbuild_python_base::records::main::fields::record_spec::record_spec; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; +use sqlbuild_python_project::compiled_views::models::NativeProjectView; +use sqlbuild_python_project::resource_views::models::NativeCompiledAudit; +use sqlbuild_python_project::resource_views::models::NativeCompiledFunction; +use sqlbuild_python_project::resource_views::models::NativeCompiledSeed; +use sqlbuild_python_project::resource_views::models::NativeCompiledSource; const MODEL_RECORD: &str = "sqlbuild._native.Model"; const MATERIALIZED: &str = "materialized"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index bdbb81f607..2132e21dd0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -17,9 +17,6 @@ use sqlbuild_rules::fix_verification::models::{FixEdit, FixEligibility}; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::ExpansionVars; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::compiled_views::relation_keys::{ - RelationKeys, project_relation_keys, -}; use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::constants::SQL_RULE_PREFIX; use crate::bindings::_helpers::rules::fix_facts::compile_facts; @@ -43,6 +40,8 @@ use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::discovery::models::NativeModelFile; use sqlbuild_python_front::render::models::SqlExpansionInputs; +use sqlbuild_python_project::compiled_views::main::compiled_models::project_relation_keys::project_relation_keys; +use sqlbuild_python_project::compiled_views::types::RelationKeys; const VERIFICATION_FAILURE: &str = "Compilation could not prove unchanged output columns, types, dependencies and lineage"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs index 6b085a37ce..746b7fc8ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs @@ -5,10 +5,10 @@ use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_rules::lint_text::models::InterpolationSite; use sqlbuild_rules::lint_text::types::PreparedSql; -use crate::bindings::_helpers::compiled_views::relation_keys; -use crate::bindings::models::Positional; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::analysis::models::Positional; +use sqlbuild_python_project::compiled_views::main::compiled_models::register_relation_keys::register_relation_keys; #[pyfunction] fn lint_sql_json(py: Python<'_>, request_json: &str) -> PyResult { @@ -63,5 +63,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?; module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?; module.add_function(wrap_pyfunction!(format_sql_json, module)?)?; - relation_keys::register(module) + register_relation_keys(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs index 794d7a499b..314baf9875 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs @@ -6,16 +6,15 @@ use std::sync::Arc; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use crate::bindings::_helpers::compiled_views::model_input_view::{ - ModelInputRecord, NativeModelInput, -}; -use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::rules::lint_pipeline::{ CompiledExpansion, CompiledSource, span_pass, }; use sqlbuild_project::models::ExpansionRecord; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::render::types::SpanRow; +use sqlbuild_python_project::compiled_views::models::NativeModelInput; +use sqlbuild_python_project::compiled_views::models::NativeProjectView; +use sqlbuild_python_project::compiled_views::types::ModelInputRecord; const FILE_PATH_ATTRIBUTE: &str = "file_path"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs index cd3b67bc3c..145f1d66f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs @@ -18,7 +18,6 @@ use sqlbuild_rules::lint_text::models::{ use crate::bindings::_helpers::rules::lint_pipeline::{ LintViolationRow, NATIVE_ENGINE, ViolationClasses, project_relative_path, }; -use crate::bindings::models::Positional; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; @@ -27,6 +26,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::analysis::models::Positional; const FORMAT_UNSAFE_CODE: &str = "format-unsafe"; const FORMAT_UNSAFE_REMEDIATION: &str = "Rewrite the unsupported SQL shape or report a minimal formatter reproduction. No header, SQL body, or fixture changes were written to this file."; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 94eaec521a..f7f185d771 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -28,13 +28,6 @@ use sqlbuild_rules::models::{ ModelRowsContext, RowsEvaluation, RowsRequest, RulesConfig, }; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; -use crate::bindings::_helpers::compiled_project::values::plain_value; -use crate::bindings::_helpers::compiled_views::audit_views::NativeCompiledAudit; -use crate::bindings::_helpers::compiled_views::model_reads::native_model; -use crate::bindings::_helpers::compiled_views::records::compiled_project_facts; -use crate::bindings::_helpers::compiled_views::records::declaration_rows; -use crate::bindings::_helpers::compiled_views::relation_keys::project_relation_keys; use crate::bindings::_helpers::rules::config_json::asdict_json; use crate::bindings::_helpers::rules::early_builtin_rules::EarlyBuiltinRules; use crate::bindings::_helpers::rules::fact_replay::{project_attribute, project_models}; @@ -53,6 +46,13 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::type_system::main::types_equal::types_equal; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_front::scopes::main::reports::metadata_projection_value::metadata_projection_value; +use sqlbuild_python_project::compiled_project::main::plain_value::plain_value; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; +use sqlbuild_python_project::compiled_views::main::compiled_models::native_model::native_model; +use sqlbuild_python_project::compiled_views::main::compiled_models::project_relation_keys::project_relation_keys; +use sqlbuild_python_project::compiled_views::main::project::compiled_project_facts::compiled_project_facts; +use sqlbuild_python_project::compiled_views::main::project::declaration_rows::declaration_rows; +use sqlbuild_python_project::resource_views::models::NativeCompiledAudit; const DATACLASSES_MODULE: &str = "dataclasses"; const NATIVE_RULES_CACHE_FILE: &str = "native.json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index 99c2f570e4..c664257c77 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -16,7 +16,6 @@ use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::rules::catalogue::{ resolved_ignore_selectors, rule_catalogue, select_rules, }; @@ -53,6 +52,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; use sqlbuild_rules::models::RowsRequest; use std::sync::Arc; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index 4c6143f6ab..7077efccbb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -7,7 +7,6 @@ use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_rules::project_tree::main::pure_path_posix::pure_path_posix; use std::path::Path; -use crate::bindings::_helpers::compiled_views::records::project_with; use crate::bindings::_helpers::rules::catalogue::{ references_custom_rules, resolved_ignore_selectors, rule_catalogue, select_rules, verify_custom_rules, @@ -23,6 +22,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::project::project_with::project_with; const MODEL_RESOURCE_TYPE: &str = "model"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index c0f7a4e6fb..6f6cd2a808 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -19,11 +19,6 @@ use sqlbuild_rules::rule_results::main::duplicate_literal_hints::duplicate_liter use sqlbuild_rules::rule_results::models::RuleFinding; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::ExpansionVars; -use crate::bindings::_helpers::compiled_views::compiled_model_view::{ - NativeCompiledModel, SqlRuleInputs, SqlRuleKey, -}; -use crate::bindings::_helpers::compiled_views::model_reads::output_proven; -use crate::bindings::_helpers::compiled_views::relation_keys::project_relation_keys; use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::constants::SQL_RULE_PREFIX; use crate::bindings::_helpers::rules::early_sql_lint::{ @@ -53,6 +48,11 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::compiled_models::output_proven::output_proven; +use sqlbuild_python_project::compiled_views::main::compiled_models::project_relation_keys::project_relation_keys; +use sqlbuild_python_project::compiled_views::models::{ + NativeCompiledModel, SqlRuleInputs, SqlRuleKey, +}; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs index 4e2021bb60..88c31f05a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_test_objects.rs @@ -10,12 +10,12 @@ use sqlbuild_rules::models::{ SqlTestParameterFact, SqlTestParameterValueFact, TestedResource, }; -use crate::bindings::_helpers::compiled_project::values::plain_value; -use crate::bindings::_helpers::compiled_views::records::plain_typed_value; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_project::main::plain_value::plain_value; +use sqlbuild_python_project::compiled_views::main::project::plain_typed_value::plain_typed_value; const DESCRIPTION_KEY: &str = "description"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index 11983dd763..e7c239a10b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -27,7 +27,6 @@ use crate::bindings::_helpers::command_runtime::connection::resolve_project_conn use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::pipeline_entries::run_compile_only_pipeline; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; @@ -43,6 +42,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::contracts::main::resolve_effective_scenario_config::resolve_effective_scenario_config; use sqlbuild_python_front::discovery::main::failures::sql_analysis_guidance::sql_analysis_guidance; +use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; const METADATA_MODULE: &str = "importlib.metadata"; const DISTRIBUTION_NAME: &str = "sqlbuild"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs index 69f9af83f1..58eb210cb7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -30,10 +30,8 @@ use sqlbuild_python_front::scopes::main::reports::scope_list::scope_list; use sqlbuild_python_front::scopes::main::reports::scope_move_preview::scope_move_preview; use sqlbuild_python_front::scopes::main::reports::scope_report::scope_report; use sqlbuild_python_front::scopes::main::reports::serialize_scope_report::serialize_scope_report; -use sqlbuild_python_front::scopes::models::ExplainArguments; -use sqlbuild_python_front::scopes::models::ListArguments; -use sqlbuild_python_front::scopes::models::ReportArguments; use sqlbuild_python_front::scopes::models::ScopeClasses; +use sqlbuild_python_front::scopes::models::{ExplainArguments, ListArguments, ReportArguments}; const TARGET_ERROR_CODE: &str = "C960"; const BARE_USAGE: &str = "bare names work for models, sources, seeds and functions; write macros, \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index 8f9d283296..73a9a0da16 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -16,7 +16,6 @@ use crate::bindings::_helpers::compile_driver::inputs::{InputsRequest, attached_ use crate::bindings::_helpers::compile_driver::project::compile_adapter_context; use crate::bindings::_helpers::frontier::cache_directory::cache_directory; use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridged, produced}; -use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; use pyo3::IntoPyObjectExt; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -31,6 +30,7 @@ use sqlbuild_python_front::scopes::main::index::open_native_scope_index::open_na use sqlbuild_python_front::scopes::main::index::project_fingerprint::project_fingerprint; use sqlbuild_python_front::scopes::main::index::read_scope_index_cache::read_scope_index_cache; use sqlbuild_python_front::scopes::main::index::write_scope_index_cache::write_scope_index_cache; +use sqlbuild_python_project::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; const SCOPE_CACHE_DIRECTORY_NAME: &str = "declaration-scopes-v2"; const SCOPE_CACHE_FILENAME: &str = "scope-index.json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 74fc45deaa..95b3ab4c7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -22,7 +22,6 @@ use crate::bindings::_helpers::command_runtime::invocation::resolve_command_invo use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; -use crate::bindings::_helpers::compiled_project::snapshot_execution::{VerifiedRun, WorkerBuild}; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::planner_entries::command_plans::build_seed_command_plan; use crate::bindings::_helpers::planner_entries::command_plans::read_selected_seed_fingerprints; @@ -39,6 +38,8 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; +use sqlbuild_python_project::project_snapshot::types::VerifiedRun; const SUCCESS_STATUS: &str = "success"; const FAILED_STATUS: &str = "failed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs index d4b551c8d9..e39ee991e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs @@ -8,9 +8,6 @@ use sqlbuild_analysis::semantic_checks::models::{ CompletedDiagnostic, CompletionModel, CompletionRequest, SemanticDiagnostic, SemanticLocation, }; -use crate::bindings::_helpers::compiled_views::compiled_model_view::LineageInput; -use crate::bindings::_helpers::compiled_views::model_reads::{inferred_names, lineage_inputs}; -use crate::bindings::_helpers::compiled_views::records::with_binding_diagnostics; use crate::bindings::_helpers::semantic_checks::failures::semantic_error; use crate::bindings::_helpers::semantic_checks::metadata::{ShapeRows, shape_rows}; use crate::bindings::_helpers::semantic_checks::session_facts::{ @@ -20,10 +17,14 @@ use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; use crate::bindings::_helpers::semantic_checks::type_recovery::{ StageShared, diagnostic_ids, id_of, items, lineage_outputs, }; -use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_names::inferred_names; +use sqlbuild_python_project::compiled_views::main::compiled_models::lineage_inputs::lineage_inputs; +use sqlbuild_python_project::compiled_views::main::project::with_binding_diagnostics::with_binding_diagnostics; +use sqlbuild_python_project::compiled_views::types::LineageInput; +use sqlbuild_python_project::semantic_shapes::main::project_shapes::project_shapes; const RECOVERED_CODE: &str = "B002"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs index 186498a7f4..aa627d60a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs @@ -3,17 +3,17 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; -use crate::bindings::_helpers::analysis::binding_catalog::ignores_quoted_case; -use crate::bindings::_helpers::analysis::schema_request::{ - SchemaRequestParts, schema_validation_request, -}; -use crate::bindings::_helpers::analysis_session::stage_functions::infer_native_query_columns; use crate::bindings::_helpers::semantic_checks::resource_sql::resource_sql_diagnostics_of; -use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; -use crate::bindings::models::ProjectCatalog; use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::analysis::main::ignores_quoted_case::ignores_quoted_case; +use sqlbuild_python_project::analysis::main::schema_validation_request::schema_validation_request; +use sqlbuild_python_project::analysis::models::ProjectCatalog; +use sqlbuild_python_project::analysis::models::SchemaRequestParts; +use sqlbuild_python_project::analysis_session::main::infer_native_query_columns::infer_native_query_columns; +use sqlbuild_python_project::semantic_shapes::main::declared_types::declared_types; +use sqlbuild_python_project::semantic_shapes::main::function_names::function_names; /// The column names a metadata config value lists: a name, or the names of a list or tuple. pub(crate) fn column_names(value: &Bound<'_, PyAny>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index ae576d3f91..ae16dd8d12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -13,15 +13,12 @@ use sqlbuild_analysis::semantic_checks::models::{ use sqlbuild_analysis::semantic_validation::models as validation; use sqlbuild_model_config::header_metadata::constants::ACCEPTED_VALUES_ARGUMENT; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::_helpers::compiled_views::model_reads::binding_validated; use crate::bindings::_helpers::semantic_checks::failures::semantic_error; use crate::bindings::_helpers::semantic_checks::inputs::{ column_names, resource_sql_diagnostics, sql_test_columns, }; use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; use crate::bindings::_helpers::semantic_checks::type_recovery::items; -use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; use sqlbuild_model_config::config_values::models::ConfigNode; use sqlbuild_project::models::ModelMetadataTexts; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; @@ -32,6 +29,9 @@ use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_front::model_config::models::AuditInstance; use sqlbuild_python_front::model_config::types::HeaderValue; use sqlbuild_python_front::model_config::types::PyConfigNode; +use sqlbuild_python_project::compiled_views::main::compiled_models::binding_validated::binding_validated; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; +use sqlbuild_python_project::semantic_shapes::main::project_shapes::project_shapes; /// `SqlReferenceKind.UDF` and `SqlReferenceKind.TABLE_FUNCTION` values. const FUNCTION_REFERENCE_KINDS: [&str; 2] = ["udf", "table_fn"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index fd8b1f3ed5..4ff5851ff8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -13,9 +13,6 @@ use sqlbuild_analysis::semantic_checks::models::{ ResourceSqlFailure, ResourceSqlKind, ResourceSqlRequest, SqlTestSqlInput, }; -use crate::bindings::_helpers::compiled_views::audit_views::audit_reference_rows; -use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; -use crate::bindings::models::ProjectCatalog; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -26,6 +23,9 @@ use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::analysis::models::ProjectCatalog; +use sqlbuild_python_project::compiled_views::models::NativeCompiledModel; +use sqlbuild_python_project::resource_views::main::audit_views::audit_reference_rows::audit_reference_rows; const SQL_ANALYSIS_CONFIG_KEY: &str = "sql_analysis"; const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs index 467c2e4eb5..011a26181c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/session_facts.rs @@ -3,7 +3,7 @@ use sqlbuild_analysis::assembly::analysis_session::main::finished_model_facts::finished_model_facts; use sqlbuild_analysis::assembly::analysis_session::models::{ColumnFact, SessionModelFacts}; -use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; +use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; /// The facts the finished `session` kept for model `name`, which the payload left out. pub(crate) fn session_model_facts<'a>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs index 4db00e85c4..ed96fc30a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs @@ -6,8 +6,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef use pyo3::types::{PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; -use crate::bindings::_helpers::compiled_views::model_reads::output_proven; use crate::bindings::_helpers::rules::early_builtin_rules::start_early_rules; use crate::bindings::_helpers::semantic_checks::completion::completed_diagnostics; use crate::bindings::_helpers::semantic_checks::metadata::metadata_diagnostics; @@ -15,10 +13,12 @@ use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; use crate::bindings::_helpers::semantic_checks::type_recovery::{ StageShared, items, recovered_types, }; -use crate::bindings::models::ProjectCatalog; use sqlbuild_python_base::boundary::main::native_compiler_error::native_compiler_error; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::analysis::models::ProjectCatalog; +use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; +use sqlbuild_python_project::compiled_views::main::compiled_models::output_proven::output_proven; const NO_CATALOG_MESSAGE: &str = "NativeCompilerError: native semantic completion: the project has no analysis catalog"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs index 4bfe9c7345..330bee76e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -2,7 +2,6 @@ use sqlbuild_host::entries::models::AdapterHost; -use crate::bindings::_helpers::compiled_views::records::project_with; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString}; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -11,6 +10,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_views::main::project::project_with::project_with; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs index 40e167e5d6..cf87c9b8d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs @@ -16,11 +16,6 @@ use sqlbuild_analysis::semantic_checks::models::{ use sqlbuild_analysis::semantic_checks::types::RevisedBinding; use sqlbuild_analysis::semantic_validation::models as validation; -use crate::bindings::_helpers::analysis::binding_catalog::schema_validations; -use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; -use crate::bindings::_helpers::compiled_views::compiled_model_view::LineageInput; -use crate::bindings::_helpers::compiled_views::model_reads::{inferred_names, lineage_inputs}; -use crate::bindings::_helpers::compiled_views::records::with_type_recovery; use crate::bindings::_helpers::semantic_checks::constants::SCHEMA_REVALIDATIONS; use crate::bindings::_helpers::semantic_checks::failures::{internal_error, semantic_error}; use crate::bindings::_helpers::semantic_checks::inputs::revalidation_request; @@ -28,7 +23,6 @@ use crate::bindings::_helpers::semantic_checks::session_facts::{ column_names, session_model_facts, }; use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; -use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; use sqlbuild_python_base::boundary::main::compiler_error::compiler_error; use sqlbuild_python_base::compile_driver::main::count_stage::count_stage; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -36,6 +30,13 @@ use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::main::fields::mapping_values::mapping_values; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_project::analysis::main::schema_validations::schema_validations; +use sqlbuild_python_project::analysis_session::models::NativeModelAnalysisSession; +use sqlbuild_python_project::compiled_views::main::compiled_models::inferred_names::inferred_names; +use sqlbuild_python_project::compiled_views::main::compiled_models::lineage_inputs::lineage_inputs; +use sqlbuild_python_project::compiled_views::main::project::with_type_recovery::with_type_recovery; +use sqlbuild_python_project::compiled_views::types::LineageInput; +use sqlbuild_python_project::semantic_shapes::main::project_shapes::project_shapes; const UNCHANGED: &str = "unchanged"; const UNKNOWN_RECOVERED_TYPE: &str = "UNKNOWN"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/constants.rs deleted file mode 100644 index d1ffa807bf..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/constants.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Stage-work names the semantic shape stage counts. - -/// Model queries analysed again against inspected source columns. -pub(crate) const SOURCE_REBINDING_ANALYSES: &str = "source_rebinding_analyses"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/mod.rs index 11ccbb1fdc..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/mod.rs @@ -1,7 +1 @@ -pub(crate) mod constants; -pub(crate) mod coverage; -pub(crate) mod declarations; -pub(crate) mod delta_shapes; pub(crate) mod registration; -pub(crate) mod shapes; -pub(crate) mod source_bindings; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs index 5116b8f3a8..76d860d2db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs @@ -2,10 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::semantic_shapes::{coverage, shapes, source_bindings}; +use sqlbuild_python_project::semantic_shapes::main::register_coverage::register_coverage; +use sqlbuild_python_project::semantic_shapes::main::register_shapes::register_shapes; +use sqlbuild_python_project::semantic_shapes::main::register_source_bindings::register_source_bindings; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - shapes::register(module)?; - coverage::register(module)?; - source_bindings::register(module) + register_shapes(module)?; + register_coverage(module)?; + register_source_bindings(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs index e9ef355756..a097921e9d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs @@ -14,11 +14,6 @@ use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_discovery::python_paths::main::python_test_paths::python_test_paths; use crate::bindings::_helpers::compile_driver::pipeline_entries::project_graph; -use crate::bindings::_helpers::graph::python_node_graph::{ - python_node_graph, run_selection_from_inputs, -}; -use crate::bindings::_helpers::graph::python_selection::python_node_names; -use crate::bindings::_helpers::planner_selection::unit_tests::split_project_unit_test_selectors; use sqlbuild_python_base::build_progress::main::terminal::flush::flush; use sqlbuild_python_base::build_progress::main::terminal::write::write; use sqlbuild_python_base::native_host::main::flush_console::flush_console; @@ -27,6 +22,10 @@ use sqlbuild_python_base::native_host::main::python_executable::python_executabl use sqlbuild_python_base::records::main::construct::new_helped_error::new_helped_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; +use sqlbuild_python_project::graph::main::python_nodes::python_node_names::python_node_names; +use sqlbuild_python_project::graph::main::python_nodes::run_selection_from_inputs::run_selection_from_inputs; +use sqlbuild_python_project::planner_selection::main::split_project_unit_test_selectors::split_project_unit_test_selectors; /// The prefix of a Python test module's file name. const TEST_MODULE_PREFIX: &str = "test_"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index a721a2c9df..7b6a2236b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -40,7 +40,6 @@ use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_com use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runtime_target; -use crate::bindings::_helpers::compiled_project::snapshot_execution::{VerifiedRun, WorkerBuild}; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::planner_entries::command_plans::build_test_command_plan; use crate::bindings::_helpers::result_output::documents::test_execution_json; @@ -70,6 +69,8 @@ use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; +use sqlbuild_python_project::project_snapshot::types::VerifiedRun; /// Why `--since` selected each test, by inspection label. type SelectionReasons = HashMap>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs index 477491e529..fd4313cdfb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs @@ -12,7 +12,6 @@ use sqlbuild_analysis::compiler::models::{ use sqlbuild_core::panics::main::native_failure::native_failure; -use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; use sqlbuild_project::models::ReferenceRow; use sqlbuild_python_base::boundary::main::value_error::value_error; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -20,6 +19,7 @@ use sqlbuild_python_base::records::main::fields::path_posix_text::path_posix_tex use sqlbuild_python_base::records::main::fields::path_stem::path_stem; use sqlbuild_python_base::types::CompilerDetach; use sqlbuild_python_front::sqltext::models::LexicalSyntaxInput; +use sqlbuild_python_project::compiled_views::models::NativeModelInput; const MACRO_TOKEN: char = '@'; const INPUT_FAILURE: &str = "input"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs index f06ce7b343..0fba97583d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs @@ -70,7 +70,11 @@ impl<'py> Classes<'py> { assembly: record_class(py, RecordId::NativeSqlTestAssembly)?, failure: record_class(py, RecordId::NativeSqlTestFailure)?, location: record_class(py, RecordId::SourceLocation)?, - code: PyString::new(py, super::constants::SQL_TEST_HELPER_REFERENCE_CODE).into_any(), + code: PyString::new( + py, + sqlbuild_python_project::sql_test_glue::constants::SQL_TEST_HELPER_REFERENCE_CODE, + ) + .into_any(), }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs deleted file mode 100644 index 7921a369a4..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! SQL-test plan resource and severity decisions. - -pub(super) const UDF: &str = "udf"; -pub(super) const TABLE_FN: &str = "table_fn"; -pub(super) const ERROR_SEVERITY: &str = "error"; -pub(super) const FIXTURE_PREFIXES: [(&str, &str); 4] = [ - ("model", "__ref__"), - ("source", "__source__"), - ("seed", "__seed__"), - ("sql_test", "__expected__"), -]; -pub(super) const CURSOR_ALGEBRA_EXCEPTIONS: &str = "sqlbuild.cursor_algebra.exceptions"; -pub(super) const INTEGER_CURSOR_TYPE: &str = "integer"; -pub(super) const TIMESTAMP_CURSOR_TYPE: &str = "timestamp"; -pub(crate) const DEFAULT_TIMESTAMP_START: &str = "1900-01-01 00:00:00"; -pub(crate) const DEFAULT_TIMESTAMP_END: &str = "2999-12-31 00:00:00"; -pub(crate) const DEFAULT_INTEGER_START: &str = "-1000000000000000"; -pub(crate) const DEFAULT_INTEGER_END: &str = "1000000000000000"; -pub(super) const CURSOR_START_KEY: &str = "cursor_start"; -pub(super) const CURSOR_END_KEY: &str = "cursor_end"; -pub(super) const SENTINEL: &str = "__SQLBUILD_CALL_SUFFIX__"; -/// The diagnostic code of a SQL test that references a helper it may not read. -pub(crate) const SQL_TEST_HELPER_REFERENCE_CODE: &str = "P013"; -/// The scan store namespace of expanded SQL-test CTEs. -pub(crate) const SQL_TEST_CTE_SCAN_ALGORITHM: &str = "expanded-sql-test-ctes-v1"; -/// The scan store namespace of the models SQL tests expect. -pub(crate) const SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM: &str = "sql-test-expected-models-v1"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index ac3d60ae0a..a2b5a62391 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -1,17 +1,3 @@ pub(crate) mod assembly; pub(crate) mod compiled_tests; -pub(crate) mod constants; -mod cursor_window; -mod fixture_validation; -pub(crate) mod models; -pub(crate) mod plan_assembly; -pub(crate) mod plan_driver; -mod plan_entries; -mod plan_errors; -mod plan_fixtures; -mod plan_rendering; -mod plan_rows; -mod plan_values; -pub(crate) mod planning; pub(crate) mod registration; -mod types; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs deleted file mode 100644 index 75520b0694..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! Typed requests of SQL-test planning. - -use pyo3::prelude::{Bound, PyAny}; - -/// How one batch of SQL tests is planned. -#[derive(Debug)] -pub(crate) struct SqlTestPlanOptions<'py> { - pub(crate) sql_analysis_enabled: Bound<'py, PyAny>, - pub(crate) render_sql: bool, - pub(crate) include_plan: bool, - pub(crate) artifact: bool, -} - -/// How assembling SQL-test plan results validates fixtures. -#[derive(Debug)] -pub(crate) struct SqlTestAssembly<'py> { - pub(crate) sql_analysis_enabled: Bound<'py, PyAny>, - pub(crate) validate_fixtures: bool, - pub(crate) fixture_planning_context: Bound<'py, PyAny>, -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs index 8c7176fe72..52e8ff7e43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -1,28 +1,19 @@ //! Register the native SQL test planning and assembly functions. use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::wrap_pyfunction; use crate::bindings::_helpers::sql_test_glue::compiled_tests; use sqlbuild_python_front::sql_test_glue::main::register_authored_ctes::register_authored_ctes; +use sqlbuild_python_project::sql_test_glue::main::plan_compiled_sql_test_artifacts_function::plan_compiled_sql_test_artifacts_function; +use sqlbuild_python_project::sql_test_glue::main::plan_sql_tests_function::plan_sql_tests_function; +use sqlbuild_python_project::sql_test_glue::main::plan_sql_tests_natively_function::plan_sql_tests_natively_function; +use sqlbuild_python_project::sql_test_glue::main::sql_test_plan_error_messages_function::sql_test_plan_error_messages_function; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - super::plan_assembly::plan_sql_tests, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::plan_driver::plan_sql_tests_natively, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::plan_driver::plan_compiled_sql_test_artifacts, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::plan_driver::sql_test_plan_error_messages, - module - )?)?; + module.add_function(plan_sql_tests_function(module)?)?; + module.add_function(plan_sql_tests_natively_function(module)?)?; + module.add_function(plan_compiled_sql_test_artifacts_function(module)?)?; + module.add_function(sql_test_plan_error_messages_function(module)?)?; register_authored_ctes(module)?; compiled_tests::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/types.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/types.rs deleted file mode 100644 index 707f3f2975..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/types.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! SQL-test fixture validation boundary values. - -use pyo3::prelude::{Bound, PyAny}; -use std::collections::BTreeMap; - -pub(super) type ValidatedTests<'py> = (Vec>, BTreeMap>); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs index 51ec6fea5f..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs @@ -1,2 +1 @@ -pub(crate) mod nullability; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs index 5c16c08005..90771ac3c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs @@ -2,11 +2,11 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::type_system::nullability; use sqlbuild_python_base::type_system::main::register_normalization::register_normalization; +use sqlbuild_python_project::type_system::main::register_nullability::register_nullability; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { register_normalization(module)?; - nullability::register(module)?; + register_nullability(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 2b824dfc40..2eb99d1af5 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -11,7 +11,6 @@ use crate::bindings::_helpers::{ references, render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, }; -use crate::bindings::models; use sqlbuild_python_base::boundary::main::register_expected_errors::register_expected_errors; use sqlbuild_python_base::boundary::main::register_native_compiler_error::register_native_compiler_error; use sqlbuild_python_base::boundary::main::register_oracles::register_oracles; @@ -19,6 +18,10 @@ use sqlbuild_python_base::records::main::registry::capture::capture; use sqlbuild_python_base::records::main::registry::register_records::register_records; use sqlbuild_python_base::records::main::registry::verify_attributes::verify_attributes; use sqlbuild_python_base::records::main::registry::verify_tables::verify_tables; +use sqlbuild_python_project::analysis::main::register_normalization::register_normalization; +use sqlbuild_python_project::analysis::models::BindingPositions; +use sqlbuild_python_project::analysis::models::CompactAnalysisJob; +use sqlbuild_python_project::analysis::models::ProjectCatalog; #[pymodule] pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -32,7 +35,7 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { native_host::registration::register(module)?; rules::registration::register(module)?; cli::registration::register(module)?; - analysis::normalization::register(module)?; + register_normalization(module)?; analysis::queries::register(module)?; analysis::sql_tests::register(module)?; sqltext::registration::register(module)?; @@ -50,9 +53,9 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { register_native_compiler_error(module)?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; cache::registration::register(module)?; attachments::registration::register(module)?; refactor::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/mod.rs b/crates/sqlbuild-python/src/bindings/mod.rs index 072cc6b98b..668a17d0aa 100644 --- a/crates/sqlbuild-python/src/bindings/mod.rs +++ b/crates/sqlbuild-python/src/bindings/mod.rs @@ -1,6 +1,5 @@ pub(crate) mod _helpers; pub mod main; -pub(crate) mod models; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs deleted file mode 100644 index b48ee3ef23..0000000000 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ /dev/null @@ -1,131 +0,0 @@ -//! Python-facing classes and mapping requests, converted to native models at the boundary. - -use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyErr, PyResult, Python}; -use pyo3::types::PyTuple; -use pyo3::{FromPyObject, pyclass}; -use sqlbuild_analysis::semantic_validation::models as validation; -use sqlbuild_analysis::semantic_validation::models::Expansion; -use sqlbuild_analysis::semantic_validation::types::Relations; - -use sqlbuild_python_base::types::FromPositional; -use sqlbuild_python_base::types::IntoPositional; -use std::collections::HashMap; - -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct CatalogInput { - pub(crate) dialect: String, - pub(crate) quoted_ignore_case: bool, - pub(crate) known_functions: Vec, - pub(crate) known_types: Vec, - #[pyo3(from_py_with = crate::bindings::_helpers::analysis::conversions::relations)] - pub(crate) relations: Relations, -} - -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct NormalizationInput { - pub(crate) sql: String, - pub(crate) dialect: String, - pub(crate) stubs: HashMap, - pub(crate) placeholders: HashMap, -} - -#[derive(FromPyObject, Debug)] -#[pyo3(from_item_all)] -pub(crate) struct PositionInput { - pub(crate) authored: String, - pub(crate) query: String, - pub(crate) expanded: String, - pub(crate) cleaned: String, - pub(crate) passes: Vec>>, -} - -#[pyclass(module = "sqlbuild._native")] -#[derive(Debug)] -pub(crate) struct ProjectCatalog { - pub(crate) inner: validation::ProjectCatalog, -} - -/// One resolved compact analysis batch that runs without borrowing its project catalog. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct CompactAnalysisJob { - pub(crate) inner: validation::CompactAnalysisJob, -} - -#[pyclass(module = "sqlbuild._native")] -#[derive(Debug)] -pub(crate) struct BindingPositions { - pub(crate) inner: validation::BindingPositions, -} - -impl From for validation::CatalogInput { - fn from(input: CatalogInput) -> Self { - Self { - dialect: input.dialect, - quoted_ignore_case: input.quoted_ignore_case, - known_functions: input.known_functions, - known_types: input.known_types, - relations: input.relations, - } - } -} - -impl From for validation::NormalizationInput { - fn from(input: NormalizationInput) -> Self { - Self { - sql: input.sql, - dialect: input.dialect, - stubs: input.stubs, - placeholders: input.placeholders, - } - } -} - -impl From for validation::PositionInput { - fn from(input: PositionInput) -> Self { - Self { - authored: input.authored, - query: input.query, - expanded: input.expanded, - cleaned: input.cleaned, - passes: input - .passes - .into_iter() - .map(Positional::unwrap_all) - .collect(), - } - } -} - -/// A record of another crate exchanged with Python as its positional tuple. -#[derive(Clone, Debug)] -pub(crate) struct Positional(pub(crate) T); - -impl Positional { - /// Each record of `items`, for Python. - pub(crate) fn wrap_all(items: Vec) -> Vec { - items.into_iter().map(Self).collect() - } - - /// Each record Python sent in `items`. - pub(crate) fn unwrap_all(items: Vec) -> Vec { - items.into_iter().map(|item| item.0).collect() - } -} - -impl<'py, T: FromPositional> FromPyObject<'py> for Positional { - fn extract_bound(value: &Bound<'py, PyAny>) -> PyResult { - T::from_positional(value).map(Self) - } -} - -impl<'py, T: IntoPositional> IntoPyObject<'py> for Positional { - type Target = PyTuple; - type Output = Bound<'py, PyTuple>; - type Error = PyErr; - - fn into_pyobject(self, py: Python<'py>) -> Result { - self.0.into_positional(py) - } -} diff --git a/fensu.toml b/fensu.toml index f6d3b5122e..7243c0573f 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1449,13 +1449,9 @@ paths = ["crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_ reason = "Each turns the one Python exception its own call can expect into None; the calls and the exceptions they accept differ." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs"] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs", "crates/sqlbuild-python-project/src/references/_helpers/relations.rs"] reason = "Each builds one staying-Python object from its own keyword set." -[[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_keys.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs"] -reason = "Each calls a class from its own lookup, a native attribute or a host entry, with keyword pairs." - [[dupes.allowlist]] paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs", "crates/sqlbuild-python-base/src/records/main/construct/new_record.rs"] reason = "The public entry forwards to the crate-internal constructor that the crate's own helpers call." @@ -1469,11 +1465,11 @@ paths = ["crates/sqlbuild-python-base/src/boundary/models.rs"] reason = "Each fills a missing keyword argument with its own default, None or False." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs"] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/inputs_view.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs"] reason = "Each compiled view reads its own field set from the Python boundary." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs"] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs", "crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_execution.rs"] reason = "Each callback or worker report forwards one distinct build or check event with its own payload." [[dupes.allowlist]] @@ -1676,8 +1672,8 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 169893 } -reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out. Raised from 169863: replace_key plan-entry plumbing and the A012 refusal, +31 with sqlbuild-python-base." +thresholds = { max_crate_code_lines = 147491 } +reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/src/bindings/_helpers"] @@ -1794,6 +1790,78 @@ path = "crates/sqlbuild-python-front/src/render/types.rs" symbols = ["StateTuple"] reason = "Python passes the expansion state as one five-item tuple; StateInput reads it in a single tuple extraction and names each position, as the positional record it replaces did." +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-project/src/analysis_session/types.rs" +symbols = ["PivotRequestTuple"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-project/src/compiled_project/types.rs" +symbols = ["FunctionSettingsTuple", "TestBlockTuple"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-project/src/compiled_views/types.rs" +symbols = ["RelationKeyTuple"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-project/src/analysis_session/models.rs" +symbols = ["FamilyRow::extract_bound"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-project/src/compiled_project/models.rs" +symbols = ["FunctionRow::extract_bound", "TestFileRow::extract_bound", "ScenarioRow::extract_bound", "AuditBlockRow::extract_bound", "AuditFileRow::extract_bound", "PythonResourceRow::extract_bound", "ScenarioDescriptionRow::extract_bound", "HookDescriptionRow::extract_bound", "ProofPy::extract_bound", "AnalysisPy::extract_bound"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-project/src/compiled_views/models.rs" +symbols = ["AnalysisProjections", "CompiledProjections", "ModelInputHandles", "InputProjections", "NativeProjectView", "SchemaHandles"] +reason = "AnalysisProjections, CompiledProjections, ModelInputHandles, InputProjections, NativeProjectView implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; SchemaHandles implements it manually so the user-authored audit arguments its specs carry stay out of logs; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-project/src/graph/models.rs" +symbols = ["CompiledProjectGraph"] +reason = "CompiledProjectGraph implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-project/src/graph/models.rs" +symbols = ["ResourceRow::extract_bound"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-project/src/planner_scope/models.rs" +symbols = ["RetainedScope"] +reason = "RetainedScope implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-project/src/planner_selection/models.rs" +symbols = ["Carrier"] +reason = "Carrier implements Debug manually because it holds the interpreter token, which does not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-project/src/resource_views/models.rs" +symbols = ["AuditProjections", "FunctionProjections", "NativeCompiledFunction", "SeedHandles", "NativeCompiledSeed", "SourceProjections"] +reason = "AuditProjections, FunctionProjections, NativeCompiledFunction, SeedHandles, NativeCompiledSeed, SourceProjections implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS012" +path = "crates/sqlbuild-python-project/src/resource_views/models.rs" +symbols = ["SeedNamespace::extract_bound"] +reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." + [[targets.rust.rule_exceptions]] rule = "FPRSR307" path = "crates/sqlbuild-rules/src/scope_metadata/main.rs" @@ -1816,7 +1884,7 @@ reason = "ProjectCatalog implements Debug manually because Polyglot SchemaValida [[targets.rust.rule_exceptions]] rule = "FPRSS201" -path = "crates/sqlbuild-python/src/bindings/models.rs" +path = "crates/sqlbuild-python-project/src/analysis/models.rs" reason = "CompactAnalysisJob holds a prepared analysis closure, which cannot implement Debug; other models derive it." [[targets.rust.rule_exceptions]] @@ -1955,6 +2023,7 @@ crate_names = [ "sqlbuild-pysource", "sqlbuild-python-base", "sqlbuild-python-front", + "sqlbuild-python-project", "sqlbuild-python", "sqlbuild-recall", "sqlbuild-refactor", @@ -2026,6 +2095,7 @@ edge_paths = [ "crates/sqlbuild-python-base/src/records/main/registry/native_attr.rs", "crates/sqlbuild-python-base/src/warehouse_access", "crates/sqlbuild-python-front/src/native_host", + "crates/sqlbuild-python-project/src/native_host", "crates/sqlbuild-python/src/bindings/_helpers/native_host", ] pending_budgets = [ @@ -2066,85 +2136,120 @@ pending_budgets = [ "crates/sqlbuild-python-base/src/records/_helpers/native_classes/registration.rs=26", "crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs=2", "crates/sqlbuild-python-base/src/type_system/_helpers/equality.rs=3", - "crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs=1", - "crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs=3", + "crates/sqlbuild-python-front/src/attachments/_helpers/cursor_intrinsics.rs=0", + "crates/sqlbuild-python-front/src/attachments/_helpers/reference_markers.rs=2", + "crates/sqlbuild-python-front/src/attachments/main/assert_no_unresolved_sql_markers.rs=1", + "crates/sqlbuild-python-front/src/attachments/main/compile_input_error.rs=1", "crates/sqlbuild-python-front/src/authored_values/_helpers/fields.rs=1", "crates/sqlbuild-python-front/src/authored_values/_helpers/project_vars.rs=3", "crates/sqlbuild-python-front/src/authored_values/_helpers/python_nodes.rs=3", - "crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs=10", + "crates/sqlbuild-python-front/src/authored_values/_helpers/sql_values.rs=4", + "crates/sqlbuild-python-front/src/authored_values/main/sql_value.rs=1", + "crates/sqlbuild-python-front/src/authored_values/main/sql_value_error.rs=5", "crates/sqlbuild-python-front/src/authored_values/models.rs=2", - "crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs=4", + "crates/sqlbuild-python-front/src/compile_attachment/_helpers/effective_config.rs=1", + "crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_settings.rs=3", "crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs=2", "crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs=2", - "crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs=9", - "crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs=2", + "crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs=3", + "crates/sqlbuild-python-front/src/contracts/main/resolve_effective_collection_rendering.rs=1", + "crates/sqlbuild-python-front/src/contracts/main/resolve_effective_scenario_config.rs=5", + "crates/sqlbuild-python-front/src/cursor/_helpers/durations.rs=0", "crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs=1", - "crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs=7", - "crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs=13", + "crates/sqlbuild-python-front/src/cursor/_helpers/overrides.rs=0", + "crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs=11", "crates/sqlbuild-python-front/src/cursor/_helpers/values.rs=4", + "crates/sqlbuild-python-front/src/cursor/main/durations/parse_cursor_duration.rs=1", + "crates/sqlbuild-python-front/src/cursor/main/durations/shifted_moment.rs=1", + "crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_integer.rs=3", + "crates/sqlbuild-python-front/src/cursor/main/overrides/cursor_timestamp.rs=4", + "crates/sqlbuild-python-front/src/cursor/main/scalars/parse_value.rs=2", "crates/sqlbuild-python-front/src/cursor/models.rs=5", "crates/sqlbuild-python-front/src/discovery/_helpers/failures/faults.rs=2", "crates/sqlbuild-python-front/src/discovery/_helpers/failures/raised.rs=9", "crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_declarations.rs=5", "crates/sqlbuild-python-front/src/discovery/_helpers/headers/header_values.rs=1", - "crates/sqlbuild-python-front/src/discovery/_helpers/headers/hook_entries.rs=2", + "crates/sqlbuild-python-front/src/discovery/_helpers/headers/hook_entries.rs=1", "crates/sqlbuild-python-front/src/discovery/_helpers/project/locations.rs=4", "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_config.rs=16", - "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs=6", + "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_facts.rs=5", "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_files.rs=3", - "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs=6", + "crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs=0", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs=3", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_providers.rs=14", - "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs=9", + "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs=7", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs=15", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_hook_files.rs=1", - "crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs=8", + "crates/sqlbuild-python-front/src/discovery/_helpers/records/declaration_records.rs=6", "crates/sqlbuild-python-front/src/discovery/_helpers/records/model_records.rs=2", - "crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs=5", - "crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs=4", + "crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs=2", + "crates/sqlbuild-python-front/src/discovery/_helpers/records/yaml_records.rs=3", "crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/scope_snapshot.rs=1", - "crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs=4", + "crates/sqlbuild-python-front/src/discovery/_helpers/snapshots/snapshot.rs=3", "crates/sqlbuild-python-front/src/discovery/_helpers/sources/sql_test_blocks.rs=2", "crates/sqlbuild-python-front/src/discovery/_helpers/sources/yaml_entries.rs=14", + "crates/sqlbuild-python-front/src/discovery/main/headers/hook_entries.rs=1", + "crates/sqlbuild-python-front/src/discovery/main/project/project_facts.rs=1", + "crates/sqlbuild-python-front/src/discovery/main/project/validation_error.rs=6", + "crates/sqlbuild-python-front/src/discovery/main/python_files/discover_materialization_files.rs=1", + "crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_node_functions.rs=1", + "crates/sqlbuild-python-front/src/discovery/main/records/discover_adapter_file.rs=2", + "crates/sqlbuild-python-front/src/discovery/main/records/discover_scenario_files.rs=1", + "crates/sqlbuild-python-front/src/discovery/main/records/discover_schema_files.rs=1", + "crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs=2", + "crates/sqlbuild-python-front/src/discovery/main/snapshots/escaped.rs=1", "crates/sqlbuild-python-front/src/macro_bridge/_helpers/module_digests.rs=2", - "crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs=26", - "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs=5", + "crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs=23", + "crates/sqlbuild-python-front/src/macro_bridge/main/context_store_token.rs=2", + "crates/sqlbuild-python-front/src/macro_bridge/main/macro_store_token.rs=1", + "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_classes.rs=3", "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_errors.rs=1", "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_templates.rs=4", "crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs=2", - "crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs=4", + "crates/sqlbuild-python-front/src/model_config/_helpers/config/template_data.rs=0", "crates/sqlbuild-python-front/src/model_config/_helpers/config/typed_config.rs=1", "crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_columns.rs=1", "crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_metadata.rs=1", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/described_models.rs=1", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/unicode_environment.rs=3", + "crates/sqlbuild-python-front/src/model_config/main/config/config_fields.rs=1", + "crates/sqlbuild-python-front/src/model_config/main/config/expanded_effective_vars.rs=1", + "crates/sqlbuild-python-front/src/model_config/main/config/run_id.rs=3", + "crates/sqlbuild-python-front/src/model_config/main/config/view.rs=1", "crates/sqlbuild-python-front/src/model_config/models.rs=2", "crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs=6", "crates/sqlbuild-python-front/src/refactor/_helpers/python_strings.rs=1", - "crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs=11", + "crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs=8", + "crates/sqlbuild-python-front/src/refactor/main/model_facts.rs=3", "crates/sqlbuild-python-front/src/render/_helpers/calls/macro_arguments.rs=4", "crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_classes.rs=1", "crates/sqlbuild-python-front/src/render/_helpers/declarations/authored_sql.rs=2", "crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_expansion.rs=24", "crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_indexes.rs=1", - "crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs=12", + "crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs=7", "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs=14", "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs=6", "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion_result.rs=4", "crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_generated_references.rs=9", + "crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs=3", + "crates/sqlbuild-python-front/src/render/main/declarations/declaration_scope_resolver.rs=2", "crates/sqlbuild-python-front/src/render/models.rs=10", "crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_tolerant.rs=5", - "crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs=2", + "crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs=1", "crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_visibility.rs=5", "crates/sqlbuild-python-front/src/scopes/_helpers/declarations/declaration_resolution.rs=15", "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs=2", "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_index.rs=24", - "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs=8", + "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs=6", "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs=18", "crates/sqlbuild-python-front/src/scopes/_helpers/relationships/compile_usages.rs=4", "crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_errors.rs=3", - "crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs=23", - "crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs=2", + "crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_report_objects.rs=20", + "crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs=1", + "crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs=1", + "crates/sqlbuild-python-front/src/scopes/main/index/build_scope_lookup.rs=2", + "crates/sqlbuild-python-front/src/scopes/main/reports/json_to_python.rs=3", + "crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs=1", "crates/sqlbuild-python-front/src/scopes/models.rs=13", "crates/sqlbuild-python-front/src/sql_test_glue/_helpers/authored_ctes.rs=2", "crates/sqlbuild-python-front/src/sqltext/_helpers/authored_values.rs=3", @@ -2153,19 +2258,92 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/sqltext/models.rs=1", "crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs=1", "crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs=4", - "crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/timestamps.rs=4", - "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs=7", - "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs=1", + "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs=6", + "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs=0", "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs=5", "crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=2", + "crates/sqlbuild-python-front/src/state_rows/main/event_rows/date_time_typed.rs=1", + "crates/sqlbuild-python-front/src/state_rows/main/event_rows/in_utc.rs=2", + "crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs=1", + "crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs=1", + "crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs=1", + "crates/sqlbuild-python-project/src/analysis/_helpers/binding_catalog.rs=0", + "crates/sqlbuild-python-project/src/analysis/main/ignores_quoted_case.rs=1", + "crates/sqlbuild-python-project/src/analysis/models.rs=1", + "crates/sqlbuild-python-project/src/analysis_session/_helpers/nullability_rules.rs=3", + "crates/sqlbuild-python-project/src/analysis_session/_helpers/profile_inputs.rs=0", + "crates/sqlbuild-python-project/src/analysis_session/main/nullability_rules.rs=1", + "crates/sqlbuild-python-project/src/compile_attachment/_helpers/cursor_roles.rs=0", + "crates/sqlbuild-python-project/src/compile_attachment/main/resolve_cursor_input_roles.rs=1", + "crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs=4", + "crates/sqlbuild-python-project/src/compiled_project/models.rs=2", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_analysis.rs=9", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs=0", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs=3", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs=9", + "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/defaulted_destination.rs=1", + "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/object_key.rs=1", + "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relation_location.rs=1", + "crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_model_name.rs=2", + "crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs=2", + "crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs=1", + "crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs=2", + "crates/sqlbuild-python-project/src/compiled_views/main/project/with_binding_diagnostics.rs=1", + "crates/sqlbuild-python-project/src/compiled_views/main/project/with_type_recovery.rs=1", + "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs=3", + "crates/sqlbuild-python-project/src/contracts/main/promotion_conflict_diagnostics.rs=3", + "crates/sqlbuild-python-project/src/fingerprints/_helpers/latest_rows.rs=13", + "crates/sqlbuild-python-project/src/fingerprints/_helpers/query_hashes.rs=1", + "crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs=16", + "crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_sql.rs=1", + "crates/sqlbuild-python-project/src/fingerprints/main/row_fingerprint.rs=2", + "crates/sqlbuild-python-project/src/graph/_helpers/nodes/algorithms.rs=3", + "crates/sqlbuild-python-project/src/graph/_helpers/nodes/execution_gates.rs=1", + "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs=0", + "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs=3", + "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs=5", + "crates/sqlbuild-python-project/src/graph/main/nodes/model_python_hook_names.rs=1", + "crates/sqlbuild-python-project/src/graph/main/nodes/sql_ref_key.rs=1", + "crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs=2", + "crates/sqlbuild-python-project/src/graph/models.rs=2", + "crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs=1", + "crates/sqlbuild-python-project/src/lineage/main/facts_eq.rs=2", + "crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs=1", + "crates/sqlbuild-python-project/src/planner_phases/main/member.rs=1", + "crates/sqlbuild-python-project/src/planner_phases/main/planner_error.rs=1", + "crates/sqlbuild-python-project/src/planner_scope/_helpers/sources.rs=2", + "crates/sqlbuild-python-project/src/planner_selection/_helpers/carrier.rs=6", + "crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs=3", + "crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs=1", + "crates/sqlbuild-python-project/src/planner_selection/models.rs=3", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs=2", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_reads.rs=4", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs=2", + "crates/sqlbuild-python-project/src/planner_strategy/main/complete_empty_fixture_sql.rs=1", + "crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs=18", + "crates/sqlbuild-python-project/src/references/_helpers/reference_facts.rs=5", + "crates/sqlbuild-python-project/src/references/_helpers/relations.rs=2", + "crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs=0", + "crates/sqlbuild-python-project/src/references/main/hard_coded_relation_remedy.rs=1", + "crates/sqlbuild-python-project/src/references/main/render_source_relation.rs=2", + "crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs=2", + "crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs=2", + "crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs=0", + "crates/sqlbuild-python-project/src/resource_views/main/seed_views/paired_seeds.rs=1", + "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/coverage.rs=1", + "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs=3", + "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs=13", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs=2", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/fixture_validation.rs=4", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs=1", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_errors.rs=4", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_rendering.rs=2", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_values.rs=2", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/planning.rs=1", + "crates/sqlbuild-python-project/src/sql_test_glue/main/planned_rows.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/nullability_rules.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs=1", @@ -2204,7 +2382,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs=2", @@ -2247,23 +2424,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/records.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs=10", @@ -2328,10 +2492,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs=15", - "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs=3", @@ -2345,13 +2505,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs=4", @@ -2390,7 +2543,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_facts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs=11", @@ -2430,8 +2582,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs=1", @@ -2464,7 +2614,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs=2", @@ -2483,19 +2632,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=14", @@ -2532,11 +2673,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs=24", - "crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs=19", - "crates/sqlbuild-python/src/bindings/_helpers/references/reference_facts.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/references/source_relations.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs=13", @@ -2595,22 +2731,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=99", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_values.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs=5", - "crates/sqlbuild-python/src/bindings/models.rs=0", ] [targets.rust.rule_options.XSB073] @@ -2622,6 +2746,7 @@ edge_paths = [ "crates/sqlbuild-python-base/src/records/_helpers/native_classes/registry.rs", "crates/sqlbuild-python-base/src/warehouse_access", "crates/sqlbuild-python-front/src/native_host", + "crates/sqlbuild-python-project/src/native_host", "crates/sqlbuild-python/src/bindings/_helpers/native_host", ] pending_budgets = [ @@ -2682,9 +2807,30 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/scopes/models.rs=11", "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs=1", "crates/sqlbuild-python-front/src/state_rows/models.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/analysis/schema_request.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs=1", + "crates/sqlbuild-python-project/src/analysis/models.rs=9", + "crates/sqlbuild-python-project/src/analysis_session/models.rs=1", + "crates/sqlbuild-python-project/src/compiled_project/models.rs=36", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs=3", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/inputs_view.rs=24", + "crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs=26", + "crates/sqlbuild-python-project/src/compiled_views/models.rs=34", + "crates/sqlbuild-python-project/src/fingerprints/_helpers/storage_io.rs=12", + "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_node_graph.rs=4", + "crates/sqlbuild-python-project/src/graph/models.rs=10", + "crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs=2", + "crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs=1", + "crates/sqlbuild-python-project/src/planner_selection/models.rs=1", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_preparation.rs=6", + "crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_sql.rs=3", + "crates/sqlbuild-python-project/src/planner_strategy/models.rs=30", + "crates/sqlbuild-python-project/src/references/_helpers/explicit_references.rs=2", + "crates/sqlbuild-python-project/src/references/_helpers/reference_facts.rs=2", + "crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs=2", + "crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs=3", + "crates/sqlbuild-python-project/src/resource_views/models.rs=33", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs=2", + "crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/planning.rs=4", + "crates/sqlbuild-python-project/src/sql_test_glue/models.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs=3", @@ -2728,17 +2874,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs=33", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs=10", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_view.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/function_views.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/inputs_view.rs=24", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs=17", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_view.rs=27", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/seed_views.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/source_views.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=0", @@ -2759,7 +2894,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs=6", @@ -2767,10 +2901,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/function_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs=22", @@ -2791,7 +2921,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=3", @@ -2812,7 +2941,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs=56", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=2", @@ -2824,12 +2952,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs=30", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs=9", @@ -2840,8 +2963,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/references/reference_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs=4", @@ -2874,10 +2995,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs=4", - "crates/sqlbuild-python/src/bindings/models.rs=0", ] [targets.rust.rule_options.XSB075] @@ -2885,25 +3002,10 @@ layer_order = ["infra", "front", "project", "plan|compile", "output", "commands" module_layers = [ "*/registration=registry", "analysis=compile", - "analysis/binding_catalog=project", - "analysis/conversions=project", - "analysis/normalization=project", - "analysis/normalization_results=project", - "analysis/relation_names=project", - "analysis/schema_request=project", "analysis_session=compile", - "analysis_session/fixture_columns=project", - "analysis_session/fixture_inferences=project", - "analysis_session/models=project", - "analysis_session/nullability_rules=project", - "analysis_session/profile_inputs=project", - "analysis_session/query_columns=project", - "analysis_session/session=project", - "analysis_session/stage_functions=project", "attachments=registry", "audit_command=commands", "auditing=compile", - "auditing/render=project", "authored_values=registry", "build_command=commands", "build_progress=output", @@ -2914,7 +3016,6 @@ module_layers = [ "clone=commands", "command_runtime=commands", "compile_attachment=compile", - "compile_attachment/cursor_roles=project", "compile_command=compile", "compile_delta=compile", "compile_driver=compile", @@ -2922,12 +3023,12 @@ module_layers = [ "compile_driver/plan_result=commands", "compile_driver/static_context=commands", "compile_outputs=compile", - "compiled_project=project", - "compiled_views=project", + "compiled_project=registry", + "compiled_views=registry", "compute_logs=output", "contract_command=commands", "consumption=plan", - "contracts=project", + "contracts=registry", "cost_command=commands", "cost_report=output", "cost_telemetry=output", @@ -2946,12 +3047,12 @@ module_layers = [ "event_exporting=output", "execution_history=front", "exported_constants=registry", - "fingerprints=project", + "fingerprints=registry", "format_command=commands", "freshness=plan", "freshness/freshness_command=commands", "frontier=compile", - "graph=project", + "graph=registry", "hand_built=commands", "hand_built/compiled_model=compile", "init_command=commands", @@ -2960,7 +3061,6 @@ module_layers = [ "lifecycle_events=plan", "lineage=compile", "lineage/lineage_command=commands", - "lineage/lineage_facts=project", "load=commands", "location_command=commands", "macro_bridge=registry", @@ -2968,9 +3068,7 @@ module_layers = [ "microbatches=plan", "model_config=registry", "native_host=front", - "native_host/model_lookup=project", "native_host/orchestration_sensors=commands", - "native_host/orchestration_triggers=project", "native_host/resource_costs_state=front", "native_host/warehouse_metadata=commands", "observations=plan", @@ -2980,31 +3078,16 @@ module_layers = [ "planner_changes=plan", "planner_entries=plan", "planner_identity=plan", - "planner_identity/graph=project", - "planner_identity/hashing=project", "planner_migrations=plan", "planner_phases=plan", "planner_phases/models=project", - "planner_phases/objects=project", "planner_phases/scopes=project", "planner_registration=registry", "planner_resolution=plan", "planner_scenario=plan", - "planner_scope=project", - "planner_selection=project", + "planner_scope=registry", + "planner_selection=registry", "planner_strategy=plan", - "planner_strategy/constants=project", - "planner_strategy/fixture_completion=project", - "planner_strategy/fixture_context=project", - "planner_strategy/fixture_metadata=project", - "planner_strategy/fixture_preparation=project", - "planner_strategy/fixture_reads=project", - "planner_strategy/fixture_selection=project", - "planner_strategy/fixture_sql=project", - "planner_strategy/fixture_types=project", - "planner_strategy/fixture_values=project", - "planner_strategy/models=project", - "planner_strategy/types=project", "planner_warehouse=plan", "presentation_text=registry", "project_assembly=compile", @@ -3016,8 +3099,8 @@ module_layers = [ "refactor=front", "refactor/offline_compile=commands", "refactor/refactor_command=commands", - "references=project", - "relation_naming=project", + "references=registry", + "relation_naming=registry", "render=registry", "result_output=output", "review_command=commands", @@ -3028,25 +3111,12 @@ module_layers = [ "scopes/scope_loading=compile", "seed_command=commands", "semantic_checks=compile", - "semantic_shapes=project", + "semantic_shapes=registry", "sql_test_command=commands", "sql_test_glue=compile", - "sql_test_glue/constants=infra", - "sql_test_glue/cursor_window=project", - "sql_test_glue/fixture_validation=project", - "sql_test_glue/models=project", - "sql_test_glue/plan_assembly=project", - "sql_test_glue/plan_driver=project", - "sql_test_glue/plan_entries=project", - "sql_test_glue/plan_errors=project", - "sql_test_glue/plan_fixtures=project", - "sql_test_glue/plan_rendering=project", - "sql_test_glue/plan_rows=project", - "sql_test_glue/plan_values=project", - "sql_test_glue/planning=project", "sqltext=registry", "state_rows=registry", - "type_system=project", + "type_system=registry", "workspace_commands=commands", ] pending_budgets = [ @@ -3088,7 +3158,7 @@ paths = ["crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs", "c reason = "Field lookups on unrelated value types in different crates: project configuration takes the first matching field of a built object, while rule headers take the last entry of an authored map so a repeated key overrides earlier ones." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs"] +paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_modules.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs", "crates/sqlbuild-python-project/src/planner_scope/_helpers/values.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs"] reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." [[dupes.allowlist]] @@ -3159,6 +3229,102 @@ reason = "Each reads its own Python mapping into its own native pairs." paths = ["crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs"] reason = "Each reads its own Python record's fields into its own native record." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs", "crates/sqlbuild-python-project/src/planner_identity/_helpers/graph.rs"] +reason = "Each bindings module registers its own functions and classes with the native module." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs"] +reason = "Each bindings module registers its own functions and classes with the native module." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/native_host/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs"] +reason = "Each context object takes its own resource out of its own lock and releases it the way that resource needs; only the take-under-lock shape is shared." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/model_config/models.rs", "crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs"] +reason = "Each host keeps its own first Python error and returns its own failure type; only the string read is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/authored_values/models.rs", "crates/sqlbuild-python-front/src/cursor/models.rs"] +reason = "SqlValueClasses::load and CursorSession::new each load the record classes their own conversions need; CursorSession's to_utc and utc_offset each turn the zone call's result into their own value." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/analysis_session/models.rs", "crates/sqlbuild-python-project/src/compiled_project/models.rs", "crates/sqlbuild-python-project/src/compiled_views/models.rs", "crates/sqlbuild-python-project/src/graph/models.rs", "crates/sqlbuild-python-project/src/resource_views/models.rs"] +reason = "Each positional record converts its own fields; models.rs holds only records, so these are the conversions positional_record! writes, written out once per record." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/getters.rs"] +reason = "Each matches its own native class's attribute names to its own getters; only the cast-and-dispatch shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/audit_views.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/seed_views.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/source_views.rs"] +reason = "Each borrows its own record from the compiled project owner by index; only the borrow-and-read shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs"] +reason = "Each reads an optional text value from its own Python mapping; consolidating these readers across the binding crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/models.rs"] +reason = "Each getter reads its own cached projection through the view's shared read helper." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_project/models.rs"] +reason = "Each records its own per-resource fact into its own resource list by index." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_project/models.rs"] +reason = "Each returns a view of its own resource list by index after its own bounds check." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_modules.rs", "crates/sqlbuild-python-project/src/graph/_helpers/project/project_graph.rs", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs"] +reason = "Each builds its own index from its own node list; only the index-building loop is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs"] +reason = "Each Python entry reads its own keyword arguments and calls the shared planner with its own options." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs", "crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs"] +reason = "One selects compiled models by name, the other keys a project field's nodes by destination; only the collecting loop is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/resource_views/models.rs"] +reason = "Each builds its own resource's key from its own record; only the key construction is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/project_snapshot/models.rs"] +reason = "Each worker report forwards one distinct check event with its own fields." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs"] +reason = "One returns the first model destination's database, the other collects every model destination's schema; only the loop over compiled models is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs", "crates/sqlbuild-python-project/src/compiled_views/models.rs"] +reason = "Each compiled view reads its own field set from the Python boundary." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_project/_helpers/project.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs"] +reason = "Each converts the declaration row of its own consumer into its own declaration type." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs"] +reason = "The analysis session and the compiled model view each build lineage inputs from their own row types." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/models.rs"] +reason = "The compiled model and the model input each borrow their own record from the same owner; only the borrow is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/models.rs"] +reason = "The compiled model and the model input each read the reference rows of their own record." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_project/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs"] +reason = "Each returns its own stored Python handle bound to the interpreter; only the bind-and-clone is alike." + [[dupes.allowlist]] paths = ["crates/sqlbuild-python-base/src/build_progress/main/transient_status/delayed_status.rs", "crates/sqlbuild-python-base/src/build_progress/main/transient_status/maybe_status.rs"] reason = "Each public entry forwards to its own helper with the same arguments." @@ -3168,36 +3334,292 @@ paths = ["crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs" reason = "Each public entry forwards to its own helper with the same arguments." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/python_files/imported_python_nodes.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs"] +reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/event_exporting/_helpers/scope.rs", "crates/sqlbuild-python-front/src/discovery/main/python_files/provider_by_name.rs", "crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/contract_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_visibility.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs"] +reason = "Each builds a dict keyed by a field of its own records; only the loop that fills the dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/main/python_files/register_imported_python_nodes.rs", "crates/sqlbuild-python-project/src/planner_identity/main/register_graph.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs"] reason = "Each bindings module registers its own functions and classes with the native module." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_connection_config.rs", "crates/sqlbuild-python-front/src/compile_attachment/main/build_effective_runtime_config.rs", "crates/sqlbuild-python-front/src/contracts/_helpers/spec_resolution.rs", "crates/sqlbuild-python-front/src/contracts/main/loader_destination_parts.rs", "crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs"] -reason = "Each public entry forwards to its own helper with the same arguments." +paths = ["crates/sqlbuild-python-project/src/compiled_project/main/text_value.rs", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/audit_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs"] +reason = "Each reads an optional text value from its own Python mapping; consolidating these readers across the binding crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/native_key.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs"] +reason = "Each reads a resource key's type and name into its own native key; consolidating the key readers across the binding crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_analysis_sql.rs", "crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_model_name.rs", "crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_reference_rows.rs", "crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/input_sql_macro_checks.rs"] +reason = "Each borrows its own record from the compiled project owner by index; only the borrow-and-read shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/model_config/_helpers/headers/header_families.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs"] +paths = ["crates/sqlbuild-python-front/src/model_config/main/headers/family_specs.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs"] reason = "Each reads its own Python record's fields into its own native record." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs"] -reason = "Each bindings module registers its own functions and classes with the native module." +paths = ["crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs"] +reason = "Each borrows its own record from the compiled project owner by index; only the borrow-and-read shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/native_host/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs"] -reason = "Each context object takes its own resource out of its own lock and releases it the way that resource needs; only the take-under-lock shape is shared." +paths = ["crates/sqlbuild-python-base/src/frontier/_helpers/encoder.rs", "crates/sqlbuild-python-base/src/python_values/_helpers/values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/text.rs"] +reason = "Each maps a Python `str` into its own JSON value type; the UTF-16 conversion itself is shared through `utf16_units`." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs", "crates/sqlbuild-python-base/src/warehouse_access/main/read_statistics.rs", "crates/sqlbuild-python-front/src/native_host/main/state_table_name.rs"] -reason = "Each public entry forwards to its own helper with the same arguments." +paths = ["crates/sqlbuild-python-project/src/planner_phases/main/construct.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_keys.rs"] +reason = "Each calls a class from its own lookup, a native attribute or a host entry, with keyword pairs." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/model_config/models.rs", "crates/sqlbuild-python-front/src/sqltext/_helpers/static_sql.rs"] -reason = "Each host keeps its own first Python error and returns its own failure type; only the string read is alike." +paths = ["crates/sqlbuild-python-project/src/graph/main/python_nodes/python_node_names.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_python_check_excludes.rs"] +reason = "Each builds its own index from its own node list; only the index-building loop is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/authored_values/models.rs", "crates/sqlbuild-python-front/src/cursor/models.rs"] -reason = "SqlValueClasses::load and CursorSession::new each load the record classes their own conversions need; CursorSession's to_utc and utc_offset each turn the zone call's result into their own value." +paths = ["crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs", "crates/sqlbuild-python-project/src/graph/main/project/project_graph.rs"] +reason = "Each builds its own index from its own node list; only the index-building loop is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/construct.rs", "crates/sqlbuild-python-front/src/compile_driver/_helpers/config_validation.rs", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs", "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/refusal.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs", "crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs", "crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/review_command/support.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] +reason = "Each builds its own error class's exception from a message; sharing one exception constructor across the binding crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/resource_views/main/audit_reads/always_run.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_kind.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_name.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/definition_name.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/description.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_limit.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_sql.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/meta.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/references.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/scope_deps.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/severity.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/sql_body.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/thresholds.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/arguments.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/body_sql.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/destination.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/fingerprint_destination.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/language.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/packages.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/relative_path.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/return_columns.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/returns.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/runtime_version.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/source_file_path.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/external.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/schema_entry.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_entry.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_file.rs"] +reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/oracles.rs", "crates/sqlbuild-python-base/src/cache/_helpers/native_store.rs", "crates/sqlbuild-python-front/src/authored_values/main/register_python_nodes.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_template_data.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_visibility.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_freshness.rs", "crates/sqlbuild-python-project/src/analysis/main/register_normalization.rs", "crates/sqlbuild-python-project/src/analysis_session/main/register_stage_functions.rs", "crates/sqlbuild-python-project/src/graph/main/nodes/register_algorithms.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_test_comparisons.rs", "crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/attachments/main/register_cursor_intrinsics.rs", "crates/sqlbuild-python-front/src/refactor/main/register_contract_edits.rs", "crates/sqlbuild-python-front/src/render/main/declarations/register_authored_sql.rs", "crates/sqlbuild-python-front/src/render/main/declarations/register_declaration_values.rs", "crates/sqlbuild-python-front/src/scopes/main/checks/register_scope_validation.rs", "crates/sqlbuild-python-front/src/sqltext/main/register_ddl_effects.rs", "crates/sqlbuild-python-front/src/sqltext/main/register_model_headers.rs", "crates/sqlbuild-python-front/src/state_rows/main/event_rows/register_events.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_projections.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_query_hashes.rs", "crates/sqlbuild-python-project/src/references/main/register_explicit_references.rs", "crates/sqlbuild-python-project/src/references/main/register_relations.rs", "crates/sqlbuild-python-project/src/type_system/main/register_nullability.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/replace.rs", "crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs", "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_contracts.rs", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs"] +reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/resource_views/main/resource_reads/destination.rs", "crates/sqlbuild-python-project/src/resource_views/main/resource_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/resource_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/resource_reads/references.rs", "crates/sqlbuild-python-project/src/resource_views/main/resource_reads/schema_entry.rs"] +reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/refactor/_helpers/contract_compare.rs", "crates/sqlbuild-python-project/src/contracts/_helpers/contract_diagnostics.rs", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/loop_classes.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs"] +reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs"] +reason = "Each reads the installed package version for its own output with its own fallback; only the metadata lookup is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/config.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/deps.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/enum_columns.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/inferred_columns.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/references.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/relative_path.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/schema_entry.rs"] +reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_reads.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/native_model.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/audit_reads.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/function_reads.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/seed_reads.rs", "crates/sqlbuild-python-project/src/resource_views/_helpers/source_reads.rs"] +reason = "Each casts to its own native view class and counts the fallback; only the cast-and-count shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs", "crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs", "crates/sqlbuild-python-front/src/cursor/main/intervals/partition.rs"] +reason = "Each runs its own cursor operation in a cursor session; only the session set-up and conversion are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/main/project/audits.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/compiled_models.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/functions.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/seeds.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/sources.rs", "crates/sqlbuild-python-project/src/compiled_views/main/project/sql_tests.rs"] +reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/lineage/main/facts_projected.rs", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs"] +reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/model_config/main/config/config_cursor_bound.rs", "crates/sqlbuild-python-front/src/model_config/main/config/config_integer_object.rs", "crates/sqlbuild-python-front/src/model_config/main/config/config_text.rs", "crates/sqlbuild-python-front/src/model_config/main/config/config_text_object.rs"] +reason = "Each checks one config key as its own value type; only the node lookup and error mapping are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/cursor/_helpers/intervals.rs", "crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs", "crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs"] +reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_reports.rs", "crates/sqlbuild-python-project/src/graph/main/project/register_views.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs"] +reason = "One converts a typed SQL value to plain Python values, the other to JSON-ready values; only the kind dispatch is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/graph/_helpers/project/views.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs"] +reason = "Each reads graph edges into its own key type; only the mapping walk is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/contracts/main/matching_dynamic_families.rs", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs"] +reason = "Each builds its own tuple from its own items; only the collect-into-tuple shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/main/reports/scope_explain.rs", "crates/sqlbuild-python-front/src/scopes/main/reports/scope_move_preview.rs"] +reason = "Each answers its own scope report request from the shared lookup reports; only reading the lookup is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/model_config/_helpers/config/config_values.rs", "crates/sqlbuild-python-front/src/model_config/main/config/config_str.rs"] +reason = "Each checks one config key as its own value type; only the node lookup and error mapping are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/python_calls.rs", "crates/sqlbuild-python-project/src/planner_phases/main/replace.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs"] +reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/compile_context/_helpers/input_reads.rs", "crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs", "crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/render/main/declarations/span_rows.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/failures/guidance.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs", "crates/sqlbuild-python-project/src/analysis_session/main/text_pairs.rs", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/compile_context/_helpers/diagnostic_collector.rs", "crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/planner_selection/_helpers/model_selectors.rs", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/lineage/main/facts_resource_name.rs", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs"] +reason = "Each reads its own record's arguments to fill its own fields; only the argument reading is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_module_files.rs", "crates/sqlbuild-python-front/src/discovery/main/python_files/discover_python_function_files.rs"] +reason = "Each discovers its own kind of Python file with the shared reader; only the call sequence is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/main/reports/scope_list.rs", "crates/sqlbuild-python-front/src/scopes/main/reports/scope_report.rs"] +reason = "Each answers its own scope report request from the shared lookup reports; only reading the lookup is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/contracts/main/register_spec_resolution.rs", "crates/sqlbuild-python-front/src/cursor/main/intervals/register_intervals.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_column.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/newest_move.rs"] +reason = "Each finds its own newest migration event in the state events; only the event walk is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/cursor/_helpers/values.rs", "crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs", "crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/destination.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/model_key.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/query_sql.rs"] +reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/main/index/register_scope_index.rs", "crates/sqlbuild-python-front/src/scopes/main/index/register_scope_lookup.rs", "crates/sqlbuild-python-front/src/sqltext/main/register_sql_references.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/_helpers/reports/scope_reports.rs"] +reason = "Each Python entry reads its own keyword options and calls its own scope report." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/_helpers/relationships/relationship_names.rs"] +reason = "Each scans its own SQL texts for its own scope facts; only the scan set-up is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/_helpers/records/sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/parse_sql_test_file.rs"] +reason = "Each parses its own kind of SQL test file with the shared reader; only the call sequence is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs", "crates/sqlbuild-python/src/bindings/_helpers/input_loops/source_loop.rs"] +reason = "Each crate strips a source expression to its relation name; consolidating these copies across the binding crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/graph/_helpers/nodes/execution_gates.rs", "crates/sqlbuild-python-project/src/references/_helpers/relations.rs"] +reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/planner_scope/main/edges.rs", "crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/main/project/load_local_config_file.rs", "crates/sqlbuild-python-front/src/discovery/main/project/load_project_config_file.rs"] +reason = "Each loads its own project file with the shared reader; only the call sequence is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_lookup.rs", "crates/sqlbuild-python-project/src/graph/main/project/edge_dict.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/graph/_helpers/nodes/algorithms.rs", "crates/sqlbuild-python-project/src/planner_selection/main/expand_project_upstream_keys.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs", "crates/sqlbuild-python-project/src/planner_identity/main/function_definition_hash.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/tables/state_tables.rs", "crates/sqlbuild-python-project/src/references/_helpers/source_relations.rs"] +reason = "Each builds its own error class's exception from a message; sharing one exception constructor across the binding crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/discovery/main/records/discover_constant_files.rs", "crates/sqlbuild-python-front/src/discovery/main/records/discover_enum_files.rs"] +reason = "Each discovers its own kind of project file with the shared reader; only the call sequence is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/contracts/_helpers/effective_config.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/planner_identity/main/graph_key.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/declared_rows.rs", "crates/sqlbuild-python-project/src/compiled_views/main/compiled_models/dep_rows.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/freshness.rs", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs"] +reason = "Each builds its own error class's exception from a message; sharing one exception constructor across the binding crates is a separate change." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/planner_phases/main/sorted_names.rs", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-project/src/analysis_session/_helpers/session.rs", "crates/sqlbuild-python/src/bindings/_helpers/lineage/project_schema.rs"] +reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/render/models.rs", "crates/sqlbuild-python-project/src/analysis_session/models.rs", "crates/sqlbuild-python-project/src/compiled_project/models.rs"] +reason = "Each positional record converts its own fields; models.rs holds only records, so these are the conversions positional_record! writes, written out once per record." [[dupes.allowlist]] paths = ["crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs", "crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs"] diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 77f5de4f70..3c7b02624c 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -287,6 +287,7 @@ "crates/sqlbuild-python/src/", "crates/sqlbuild-python-base/src/", "crates/sqlbuild-python-front/src/", + "crates/sqlbuild-python-project/src/", ) NATIVE_CROSSING_PATTERNS: tuple[str, ...] = ( r"\.getattr\(", diff --git a/scripts/native_module_paths/constants.py b/scripts/native_module_paths/constants.py index 2c5ea422f0..904f3f4ce6 100644 --- a/scripts/native_module_paths/constants.py +++ b/scripts/native_module_paths/constants.py @@ -5,6 +5,7 @@ SCANNED_SOURCE_ROOTS: tuple[str, ...] = ( "crates/sqlbuild-python-base/src", "crates/sqlbuild-python-front/src", + "crates/sqlbuild-python-project/src", "crates/sqlbuild-python/src", "crates/sqlbuild-cli/src", "crates/sqlbuild-host/src", diff --git a/tests/native_field_fallbacks.txt b/tests/native_field_fallbacks.txt index 4c175d640f..26f2188a9b 100644 --- a/tests/native_field_fallbacks.txt +++ b/tests/native_field_fallbacks.txt @@ -4,9 +4,9 @@ crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs BindingCatalog.prepare_analysis crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs BindingCatalog.schemas crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs .schemas -crates/sqlbuild-python/src/bindings/_helpers/analysis_session/nullability_rules.rs InferredNullability.value -crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs BindingCatalog.expression_shapes -crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs BindingCatalog.native +crates/sqlbuild-python-project/src/analysis_session/_helpers/nullability_rules.rs InferredNullability.value +crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs BindingCatalog.expression_shapes +crates/sqlbuild-python-project/src/analysis_session/_helpers/stage_functions.rs BindingCatalog.native crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs AuditCallbackRelay.on_connection_complete crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs AuditCallbackRelay.on_connection_error crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs AuditCallbackRelay.on_result @@ -107,22 +107,22 @@ crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs CompileReuseAttempt.entry_path crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs CompileReuseAttempt.native crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs CompileReuseAttempt.outcome -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs LayoutConfig.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs LayoutConfig.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs PythonHookEntry.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs PythonHookEntry.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RuleExemption.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RuleExemption.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RuleIgnore.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RuleIgnore.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RulesCacheConfig.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RulesCacheConfig.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RulesConfig.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs RulesConfig.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs SqlHookEntry.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs SqlHookEntry.__dict__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs SqlTestRulesConfig.__dataclass_fields__ -crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs SqlTestRulesConfig.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs LayoutConfig.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs LayoutConfig.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs PythonHookEntry.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs PythonHookEntry.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RuleExemption.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RuleExemption.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RuleIgnore.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RuleIgnore.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RulesCacheConfig.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RulesCacheConfig.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RulesConfig.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs RulesConfig.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs SqlHookEntry.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs SqlHookEntry.__dict__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs SqlTestRulesConfig.__dataclass_fields__ +crates/sqlbuild-python-project/src/compiled_project/_helpers/values.rs SqlTestRulesConfig.__dict__ crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs PreparedCompileArtifacts.start crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs BindingCatalog.native crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs BindingCatalog.schemas @@ -148,7 +148,8 @@ crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs CompileInp crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs PreparedCompileArtifacts.publish crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs PythonNodeGraph.selected_sql_refs crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs PlannerScopeResolution.stale_warning_scope -crates/sqlbuild-python/src/bindings/_helpers/compiled_views/records.rs CollectionRendering.value +crates/sqlbuild-python-project/src/compiled_views/_helpers/project/records.rs CollectionRendering.value +crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs CollectionRendering.value crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs ConnectionProgressReporter.on_connection_complete crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/rows.rs SqlValueKind.value crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs DbtInitProgressReporter.complete @@ -328,7 +329,7 @@ crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs Scena crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs PlannerRelationsContext.model_locations crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs PlannerRelationsContext.seed_locations crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs PlannerRelationsContext.source_read_map -crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs CompiledResourceType.value +crates/sqlbuild-python-project/src/planner_strategy/_helpers/fixtures/fixture_values.rs CompiledResourceType.value crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs PlanEntryBuildInputs.max_microbatches crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs PlanEntryBuildInputs.max_microbatches_is_override crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs PlanEntryBuildInputs.microbatch_limit_action @@ -382,7 +383,9 @@ crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs Column crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/values.rs SqlValueKind.value crates/sqlbuild-python-front/src/refactor/_helpers/refactor_facts.rs DeclarationKind.value crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs CollectionRendering.value +crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs CollectionRendering.value crates/sqlbuild-python-front/src/render/_helpers/declarations/declaration_values.rs VisibilityReason.value +crates/sqlbuild-python-front/src/render/main/declarations/constant_declaration_sql.rs VisibilityReason.value crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_classes.rs DeclarationKind.value crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs NativeMacroExpansion.render_relation crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs ResourceKind.value @@ -414,7 +417,7 @@ crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs ScopeDia crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs ScopeDiagnosticSeverity.value crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs ScopeKind.value crates/sqlbuild-python-front/src/scopes/_helpers/index/scope_objects.rs UsageKind.value -crates/sqlbuild-python-front/src/scopes/_helpers/checks/scope_validation.rs ScopeDiagnosticCode.value +crates/sqlbuild-python-front/src/scopes/main/checks/validate_scope_index.rs ScopeDiagnosticCode.value crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs ConnectionProgressReporter.on_connection_start crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs NativeProgressProjector.configure_resources crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs NativeProgressProjector.consume_resource_terminal @@ -423,14 +426,14 @@ crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs SeedConnect crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs SeedConnectionRelay.on_connection_error crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/inputs.rs BindingCatalog.native crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs BindingCatalog.native -crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs BindingCatalog.inferred_schema -crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs BindingPositions.position +crates/sqlbuild-python-project/src/semantic_shapes/_helpers/shapes.rs BindingCatalog.inferred_schema +crates/sqlbuild-python-project/src/semantic_shapes/_helpers/source_bindings.rs BindingPositions.position crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs PlanningProgressReporter.on_progress -crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs CursorGrain.value -crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs CursorType.value -crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs CompiledResourceType.value -crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs WarningSeverity.value -crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs CompiledResourceType.value +crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs CursorGrain.value +crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/cursor_window.rs CursorType.value +crates/sqlbuild-python-project/src/sql_test_glue/_helpers/inputs/fixture_validation.rs CompiledResourceType.value +crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_driver.rs WarningSeverity.value +crates/sqlbuild-python-project/src/sql_test_glue/_helpers/plan/plan_entries.rs CompiledResourceType.value crates/sqlbuild-python-front/src/state_rows/_helpers/event_rows/events.rs OldNameViewEventType.value crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_io.rs ColumnMigrationDecision.records_event crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_projections.rs OldNameViewEventType.value