Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
32 changes: 31 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ members = [
"crates/sqlbuild-project",
"crates/sqlbuild-cli",
"crates/sqlbuild-python-base",
"crates/sqlbuild-python-front",
"crates/sqlbuild-python",
]
resolver = "2"
Expand Down Expand Up @@ -57,9 +58,10 @@ order = [
"sqlbuild-project",
"sqlbuild-cli",
"sqlbuild-python-base",
"sqlbuild-python-front",
"sqlbuild-python",
]
python-boundary = ["sqlbuild-python-base", "sqlbuild-python"]
python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python"]
polyglot-floor = "sqlbuild-analysis"

[workspace.package]
Expand Down Expand Up @@ -180,6 +182,7 @@ sqlbuild-orchestration = { path = "crates/sqlbuild-orchestration" }
sqlbuild-process-ownership = { path = "crates/sqlbuild-process-ownership" }
sqlbuild-project = { path = "crates/sqlbuild-project" }
sqlbuild-python-base = { path = "crates/sqlbuild-python-base" }
sqlbuild-python-front = { path = "crates/sqlbuild-python-front" }
sqlbuild-pysource = { path = "crates/sqlbuild-pysource" }
sqlbuild-recall = { path = "crates/sqlbuild-recall" }
sqlbuild-scopes = { path = "crates/sqlbuild-scopes" }
Expand Down
5 changes: 3 additions & 2 deletions crates/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,15 @@ tested and benchmarked with `cargo test` alone.
| `sqlbuild-project` | The compiled project as plain Rust: `CompiledProject` with the producers' facts and each resource's records (model inputs and schemas, compiled models and their analysis, sources, functions, audits and seeds), and what those records answer without Python, such as lineage rows, resolved namespaces and dependency keys. Values the host owns, such as user config and discovered files, and the views it builds lazily live in one slot per record, whose types `sqlbuild-python` supplies through the `ProjectHost` trait; reads that need Python stay in the bindings as extension traits. |
| `sqlbuild-cli` | The `sqb` and `sqlbuild` executables: `--version`, unchanged compiles replayed from the shared reuse store with Python's exact streams, exit code and compute log capture, and a handoff of every other invocation to the installed `sqlbuild-python` script. Built as its own wheel (`crates/sqlbuild-cli/pyproject.toml`). |
| `sqlbuild-python-base` | The base of the PyO3 bindings: native record, enum and error classes with their specs, registration, construction and field reads; keyword calls, imports, conversions and panics at the Python boundary; host entry calls; the process allocator; and the shared runtime the binding modules build on: the native store, compile context, diagnostics, progress and terminal text, warehouse access and orchestration reads. |
| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-base`. |
| `sqlbuild-python-front` | The front of the compile pipeline as PyO3 bindings, on top of `sqlbuild-python-base`: project discovery, model configuration and headers, scopes, macro rendering and authored values, SQL text, model attachment and the compile driver, cursors, contract and refactor facts, and the warehouse state tables of consumption, observations and audit results. |
| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-front` and `sqlbuild-python-base`. |

## Layering

Dependencies point one way, from the top of this graph to the bottom:

```text
sqlbuild-python -> sqlbuild-python-base
sqlbuild-python -> sqlbuild-python-front -> sqlbuild-python-base
-> sqlbuild-project -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments
-> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host
-> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core
Expand Down
37 changes: 37 additions & 0 deletions crates/sqlbuild-python-front/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
[package]
name = "sqlbuild-python-front"
description = "The front layer of the sqlbuild._native bindings: discovery, model configuration, scopes, rendering and SQL text."
version.workspace = true
edition.workspace = true
license.workspace = true
publish.workspace = true
rust-version.workspace = true

[dependencies]
chrono.workspace = true
chrono-tz.workspace = true
pyo3.workspace = true
serde_json.workspace = true
sqlbuild-analysis.workspace = true
sqlbuild-attachments.workspace = true
sqlbuild-cache.workspace = true
sqlbuild-cli.workspace = true
sqlbuild-config.workspace = true
sqlbuild-core.workspace = true
sqlbuild-cursor.workspace = true
sqlbuild-discovery.workspace = true
sqlbuild-host.workspace = true
sqlbuild-model-config.workspace = true
sqlbuild-orchestration.workspace = true
sqlbuild-pysource.workspace = true
sqlbuild-python-base.workspace = true
sqlbuild-refactor.workspace = true
sqlbuild-render.workspace = true
sqlbuild-scopes.workspace = true
sqlbuild-sqltext.workspace = true
sqlbuild-state.workspace = true
stacker.workspace = true
uuid.workspace = true

[lints]
workspace = true
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
//! Python audit argument values read as plain argument values.

use crate::bindings::_helpers::model_config::config_nodes::{PyConfigNode, config_object};
use crate::model_config::main::config::config_object::config_object;
use crate::model_config::types::PyConfigNode;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python};
use pyo3::types::{
PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitte
use sqlbuild_attachments::ceremonial_select::models::OmittedSelect;
use sqlbuild_sqltext::sql_scan::models::LexicalSyntax;

use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput;
use crate::sqltext::models::LexicalSyntaxInput;
use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard;

/// The code-point offset to insert `SELECT 1` at, or `None`.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
//! Cursor intrinsic validation, rejection and rendering.

use crate::attachments::constants::MICROBATCH_END_SENTINEL;
use crate::attachments::constants::MICROBATCH_START_SENTINEL;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult};
use pyo3::types::{PyString, PyStringMethods};
use pyo3::{Python, pyfunction, wrap_pyfunction};
Expand All @@ -23,10 +25,7 @@ const INTEGER_ANALYSIS_BOUND: &str = "CAST(0 AS BIGINT)";
const TIMESTAMP_ANALYSIS_BOUND: &str = "CAST('2000-01-01 00:00:00' AS TIMESTAMP)";
const MODEL_SQL_CONTEXT: &str = "Model SQL";
const SQL_CONTEXT: &str = "SQL";
/// Executor placeholder replaced with a microbatch's start bound.
pub(crate) const MICROBATCH_START_SENTINEL: &str = "__SQB_CURSOR_START__";
/// Executor placeholder replaced with a microbatch's end bound.
pub(crate) const MICROBATCH_END_SENTINEL: &str = "__SQB_CURSOR_END__";

const RESERVED_CURSOR_MARKERS: [&str; 2] = [MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL];

/// The reserved executor bound markers no authored SQL may contain.
Expand All @@ -53,7 +52,7 @@ fn text_value(value: &Bound<'_, PyAny>) -> Option<String> {
}

/// `sql` with each intrinsic replaced by its bound and whether any was, raising for `context`.
fn replaced(
pub(crate) fn replaced(
py: Python<'_>,
sql: &str,
context: &str,
Expand Down
5 changes: 5 additions & 0 deletions crates/sqlbuild-python-front/src/attachments/_helpers/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pub(crate) mod argument_values;
pub(crate) mod ceremonial_select;
pub(crate) mod cursor_intrinsics;
pub(crate) mod function_headers;
pub(crate) mod reference_markers;
7 changes: 7 additions & 0 deletions crates/sqlbuild-python-front/src/attachments/constants.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
//! Public constants of attachments.

/// Executor placeholder replaced with a microbatch's start bound.
pub const MICROBATCH_START_SENTINEL: &str = "__SQB_CURSOR_START__";

/// Executor placeholder replaced with a microbatch's end bound.
pub const MICROBATCH_END_SENTINEL: &str = "__SQB_CURSOR_END__";
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
//! `analysis_cursor_sql` with each replaced call as a splice of `sql`.

use crate::attachments::_helpers::cursor_intrinsics;
use sqlbuild_analysis::sql_macros::models::TextSplice;

/// `analysis_cursor_sql` with each replaced call as a splice of `sql`.
#[inline]
pub fn analysis_cursor_splices(
sql: &str,
cursor_type: Option<&str>,
) -> Result<(String, Vec<TextSplice>), String> {
cursor_intrinsics::analysis_cursor_splices(sql, cursor_type)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
//! `sql` with intrinsics replaced by stable typed literals, only for static SQL analysis.

use crate::attachments::_helpers::cursor_intrinsics;
use pyo3::Python;
use pyo3::prelude::Bound;
use pyo3::prelude::PyAny;
use pyo3::prelude::PyResult;

/// `sql` with intrinsics replaced by stable typed literals, only for static SQL analysis.
#[inline]
pub fn analysis_cursor_sql(
py: Python<'_>,
sql: &str,
cursor_type: &Bound<'_, PyAny>,
) -> PyResult<String> {
cursor_intrinsics::analysis_cursor_sql(py, sql, cursor_type)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
//! Read a `dict[str, object]` in insertion order, leaving out keys no parameter can name.

use crate::attachments::_helpers::argument_values;
use pyo3::prelude::Bound;
use pyo3::prelude::PyResult;
use pyo3::types::PyDict;
use sqlbuild_attachments::audits::models::ArgumentValue;

/// Read a `dict[str, object]` in insertion order, leaving out keys no parameter can name.
#[inline]
pub fn argument_pairs(arguments: &Bound<'_, PyDict>) -> PyResult<Vec<(String, ArgumentValue)>> {
argument_values::argument_pairs(arguments)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
//! Fail if executable SQL still contains unresolved reference markers or an unclosed construct.

use crate::attachments::_helpers::reference_markers;
use pyo3::prelude::Bound;
use pyo3::prelude::PyResult;
use pyo3::prelude::Python;
use pyo3::types::PyString;

/// Fail if executable SQL still contains unresolved reference markers or an unclosed construct.
#[inline]
pub fn assert_no_unresolved_sql_markers(
py: Python<'_>,
sql: &Bound<'_, PyString>,
context: &str,
) -> PyResult<()> {
reference_markers::assert_no_unresolved_sql_markers(py, sql, context)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
//! A `CompileInputError` carrying `message`.

use crate::attachments::_helpers::cursor_intrinsics;
use pyo3::Python;
use pyo3::prelude::PyErr;
use pyo3::prelude::PyResult;

/// A `CompileInputError` carrying `message`.
#[inline]
pub fn compile_input_error(py: Python<'_>, message: String) -> PyResult<PyErr> {
cursor_intrinsics::compile_input_error(py, message)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
//! Native-built argument containers need no intermediate Python list or tuple.

use crate::attachments::_helpers::argument_values;
use crate::model_config::types::PyConfigNode;
use pyo3::prelude::PyResult;
use pyo3::prelude::Python;
use sqlbuild_attachments::audits::models::ArgumentValue;

/// Native-built argument containers need no intermediate Python list or tuple.
#[inline]
pub fn config_argument_value(py: Python<'_>, value: &PyConfigNode) -> PyResult<ArgumentValue> {
argument_values::config_argument_value(py, value)
}
13 changes: 13 additions & 0 deletions crates/sqlbuild-python-front/src/attachments/main/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
pub mod analysis_cursor_splices;
pub mod analysis_cursor_sql;
pub mod argument_pairs;
pub mod assert_no_unresolved_sql_markers;
pub mod compile_input_error;
pub mod config_argument_value;
pub mod parsed_header;
pub mod register_ceremonial_select;
pub mod register_cursor_intrinsics;
pub mod register_reference_markers;
pub mod reserved_cursor_markers;
pub mod resolve_cursor_intrinsics;
pub mod validated_model_cursor_sql;
17 changes: 17 additions & 0 deletions crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
//! One function file's parsed header; non-string keys and `None` values are never read.

use crate::attachments::_helpers::function_headers;
use pyo3::prelude::Bound;
use pyo3::types::PyDict;
use sqlbuild_attachments::functions::models::FunctionHeader;
use sqlbuild_attachments::functions::models::FunctionLanguage;

/// One function file's parsed header; non-string keys and `None` values are never read.
#[inline]
pub fn parsed_header(
header_values: &Bound<'_, PyDict>,
language: FunctionLanguage,
relative_path: &str,
) -> FunctionHeader {
function_headers::parsed_header(header_values, language, relative_path)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
//! Register the ceremonial select functions and classes.

use crate::attachments::_helpers::ceremonial_select;
use pyo3::prelude::Bound;
use pyo3::prelude::PyModule;
use pyo3::prelude::PyResult;

/// Register the ceremonial select functions and classes.
#[inline]
pub fn register_ceremonial_select(module: &Bound<'_, PyModule>) -> PyResult<()> {
ceremonial_select::register(module)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
//! Register the cursor intrinsics functions and classes.

use crate::attachments::_helpers::cursor_intrinsics;
use pyo3::prelude::Bound;
use pyo3::prelude::PyModule;
use pyo3::prelude::PyResult;

/// Register the cursor intrinsics functions and classes.
#[inline]
pub fn register_cursor_intrinsics(module: &Bound<'_, PyModule>) -> PyResult<()> {
cursor_intrinsics::register(module)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
//! Register the reference markers functions and classes.

use crate::attachments::_helpers::reference_markers;
use pyo3::prelude::Bound;
use pyo3::prelude::PyModule;
use pyo3::prelude::PyResult;

/// Register the reference markers functions and classes.
#[inline]
pub fn register_reference_markers(module: &Bound<'_, PyModule>) -> PyResult<()> {
reference_markers::register(module)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
//! The reserved executor bound markers no authored SQL may contain.

use crate::attachments::_helpers::cursor_intrinsics;

/// The reserved executor bound markers no authored SQL may contain.
#[inline]
pub fn reserved_cursor_markers() -> Vec<String> {
cursor_intrinsics::reserved_cursor_markers()
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
//! Whether `sql` uses cursor intrinsics, and `sql` with both bounds rendered when given.

use crate::attachments::_helpers::cursor_intrinsics;
use pyo3::Python;
use pyo3::prelude::PyResult;

/// Whether `sql` uses cursor intrinsics, and `sql` with both bounds rendered when given.
#[inline]
pub fn resolve_cursor_intrinsics(
py: Python<'_>,
sql: &str,
start_sql: Option<&str>,
end_sql: Option<&str>,
) -> PyResult<(String, bool)> {
cursor_intrinsics::resolve_cursor_intrinsics(py, sql, start_sql, end_sql)
}
Loading
Loading