Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
36 changes: 31 additions & 5 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ members = [
"crates/sqlbuild-python-front",
"crates/sqlbuild-python-project",
"crates/sqlbuild-python-planner",
"crates/sqlbuild-python-compile",
"crates/sqlbuild-python",
]
resolver = "2"
Expand Down Expand Up @@ -63,9 +64,10 @@ order = [
"sqlbuild-python-front",
"sqlbuild-python-project",
"sqlbuild-python-planner",
"sqlbuild-python-compile",
"sqlbuild-python",
]
python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python-planner", "sqlbuild-python"]
python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python-planner", "sqlbuild-python-compile", "sqlbuild-python"]
polyglot-floor = "sqlbuild-analysis"

[workspace.package]
Expand Down Expand Up @@ -189,6 +191,7 @@ sqlbuild-python-base = { path = "crates/sqlbuild-python-base" }
sqlbuild-python-front = { path = "crates/sqlbuild-python-front" }
sqlbuild-python-project = { path = "crates/sqlbuild-python-project" }
sqlbuild-python-planner = { path = "crates/sqlbuild-python-planner" }
sqlbuild-python-compile = { path = "crates/sqlbuild-python-compile" }
sqlbuild-pysource = { path = "crates/sqlbuild-pysource" }
sqlbuild-recall = { path = "crates/sqlbuild-recall" }
sqlbuild-scopes = { path = "crates/sqlbuild-scopes" }
Expand Down
5 changes: 3 additions & 2 deletions crates/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,15 @@ tested and benchmarked with `cargo test` alone.
| `sqlbuild-python-front` | The front of the compile pipeline as PyO3 bindings, on top of `sqlbuild-python-base`: project discovery, model configuration and headers, scopes, macro rendering and authored values, SQL text, model attachment and the compile driver, cursors, contract and refactor facts, and the warehouse state tables of consumption, observations and audit results. |
| `sqlbuild-python-project` | The compiled project as PyO3 bindings, on top of `sqlbuild-python-front`: the project host and the Python views of compiled models, inputs and resources, the compiled-project snapshot, analysis classes and the analysis session, the project graph, references, lineage, fingerprints, contracts and semantic shapes, audit rendering, and the planner's identities, scope, selection, phases, fixture strategy and SQL test planning. |
| `sqlbuild-python-planner` | The plan layer as PyO3 bindings, on top of `sqlbuild-python-project`: execution plans and their entries, planner phases, change detection, migrations, reference and cursor resolution, scenarios, strategy and warehouse inspection, plus freshness, consumption, microbatches, observations, lifecycle events and plan output. |
| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-planner`, `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. |
| `sqlbuild-python-compile` | The compile layer as PyO3 bindings, on top of `sqlbuild-python-project` and beside `sqlbuild-python-planner`: the compile command and driver, input loops and compile-input attachment, project assembly, analysis and the analysis session, Rules, lineage, semantic checks, compile deltas and outputs, the DAG artifact, manifest, frontier, offline scope facts, SQL test glue and auditing. |
| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-compile`, `sqlbuild-python-planner`, `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. |

## Layering

Dependencies point one way, from the top of this graph to the bottom:

```text
sqlbuild-python -> sqlbuild-python-planner -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base
sqlbuild-python -> sqlbuild-python-compile -> sqlbuild-python-planner -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base
-> sqlbuild-project -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments
-> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host
-> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core
Expand Down
37 changes: 37 additions & 0 deletions crates/sqlbuild-python-compile/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
[package]
name = "sqlbuild-python-compile"
description = "The compile layer of the sqlbuild._native bindings: the compile driver and command, input loops, project assembly, analysis and the analysis session, rules, lineage, semantic checks, compile deltas and outputs, the DAG artifact, manifest, frontier and auditing."
version.workspace = true
edition.workspace = true
license.workspace = true
publish.workspace = true
rust-version.workspace = true

[dependencies]
fensu-policy.workspace = true
pyo3.workspace = true
rayon.workspace = true
serde.workspace = true
serde_json.workspace = true
sqlbuild-analysis.workspace = true
sqlbuild-attachments.workspace = true
sqlbuild-cache.workspace = true
sqlbuild-cli.workspace = true
sqlbuild-config.workspace = true
sqlbuild-core.workspace = true
sqlbuild-discovery.workspace = true
sqlbuild-host.workspace = true
sqlbuild-model-config.workspace = true
sqlbuild-project.workspace = true
sqlbuild-pysource.workspace = true
sqlbuild-python-base.workspace = true
sqlbuild-python-front.workspace = true
sqlbuild-python-project.workspace = true
sqlbuild-render.workspace = true
sqlbuild-rules.workspace = true
sqlbuild-scopes.workspace = true
sqlbuild-sqltext.workspace = true
sqlclean.workspace = true

[lints]
workspace = true
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
//! A hand-built model's binding diagnostics at their authored locations.

use crate::analysis::main::locate_model_binding_diagnostics::locate_model_binding_diagnostics;
use crate::analysis::models::LocatedRow;
use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult};
use pyo3::wrap_pyfunction;
use sqlbuild_analysis::binding_locations::models::LocatedDiagnostic;

use sqlbuild_python_project::analysis::models::Positional;

pub(crate) fn located_row(item: LocatedDiagnostic) -> LocatedRow {
LocatedRow {
code: item.code,
message: item.message,
severity: item.severity,
line: item.line,
column: item.column,
span: item.location.map(Positional),
}
}

pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
module.add_function(wrap_pyfunction!(locate_model_binding_diagnostics, module)?)?;
Ok(())
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,9 @@
use pyo3::IntoPyObjectExt;
use pyo3::prelude::{Bound, PyAny, PyResult, Python};

use sqlbuild_python_base::records::main::registry::class_of::class_of;
use sqlbuild_python_base::records::models::GetterEntry;
use sqlbuild_python_project::analysis::models::BindingCatalog;

/// The getter readers of this module's native pyclasses.
pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry {
class: class_of::<BindingCatalog>,
read: read_binding_catalog,
}];

fn read_binding_catalog<'py>(
pub(crate) fn read_binding_catalog<'py>(
value: &Bound<'py, PyAny>,
name: &str,
) -> Option<PyResult<Bound<'py, PyAny>>> {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
pub(crate) mod binding_locations;
pub(crate) mod getters;
pub(crate) mod queries;
mod required_columns;
pub(crate) mod required_columns;
pub(crate) mod sql_test_comparisons;
pub(crate) mod sql_tests;
31 changes: 31 additions & 0 deletions crates/sqlbuild-python-compile/src/analysis/_helpers/queries.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
//! Validate, fingerprint and analyze SQL queries for the Python compiler.

use pyo3::prelude::{PyResult, Python};
use pyo3::pyfunction;

use sqlbuild_python_base::boundary::main::value_error::value_error;
use sqlbuild_python_base::types::CompilerDetach;

#[pyfunction]
pub(crate) fn query_fingerprint(py: Python<'_>, sql: &str, dialect: &str) -> PyResult<String> {
py.compiler_detach(|| {
sqlbuild_analysis::sql_tokens::main::query_fingerprint::query_fingerprint(sql, dialect)
})
.map_err(value_error)
}

/// A warehouse type name's JSON type tree, None without a parser, `ValueError` on a panic.
#[pyfunction]
pub(crate) fn parse_warehouse_type(
py: Python<'_>,
spelling: &str,
dialect: &str,
) -> PyResult<Option<String>> {
py.compiler_detach(|| {
sqlbuild_analysis::warehouse_types::main::parse_warehouse_type::parse_warehouse_type(
spelling, dialect,
)
.map(|tree| tree.as_ref().map(serde_json::Value::to_string))
})
.map_err(value_error)
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@

use std::collections::HashMap;

use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python};
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python};
use pyo3::types::{PyDict, PyDictMethods};
use pyo3::{FromPyObject, pyfunction, wrap_pyfunction};
use pyo3::{FromPyObject, pyfunction};
use serde_json::{Value, json};
use sqlbuild_analysis::compiler::main::missing_expected_columns::missing_expected_columns;
use sqlbuild_analysis::compiler::main::scenario_expected_comparison_sql::scenario_expected_comparison_sql;
Expand All @@ -22,11 +22,11 @@ const TOP_CLAUSE_DIALECT: &str = "tsql";

/// Operator, dialect and optional probed chain step of one comparison render.
pub(crate) struct RenderSettings {
set_difference_operator: String,
sql_analysis_dialect: Option<String>,
probe_step_index: Option<usize>,
output_step_index: Option<usize>,
actual_relations: HashMap<usize, String>,
pub(crate) set_difference_operator: String,
pub(crate) sql_analysis_dialect: Option<String>,
pub(crate) probe_step_index: Option<usize>,
pub(crate) output_step_index: Option<usize>,
pub(crate) actual_relations: HashMap<usize, String>,
}

impl RenderSettings {
Expand Down Expand Up @@ -59,15 +59,15 @@ fn optional_value<T: for<'py> FromPyObject<'py>>(
/// Comparison SQL of each planned SQL test, or of one chain step's actual-output probe.
#[pyfunction]
#[pyo3(signature = (test_entries, **options))]
fn render_sql_test_comparison_sql(
pub(crate) fn render_sql_test_comparison_sql(
py: Python<'_>,
test_entries: Vec<Bound<'_, PyAny>>,
options: Option<&Bound<'_, PyDict>>,
) -> PyResult<Vec<String>> {
rendered_comparisons(py, &test_entries, &RenderSettings::read(options)?)
}

fn rendered_comparisons(
pub(crate) fn rendered_comparisons(
py: Python<'_>,
test_entries: &[Bound<'_, PyAny>],
settings: &RenderSettings,
Expand Down Expand Up @@ -116,26 +116,6 @@ fn rendered_comparisons(
.collect()
}

/// The comparison SQL of one planned SQL test under `adapter`'s operator and dialect.
pub(crate) fn single_comparison_sql(
entry: &Bound<'_, PyAny>,
adapter: &Bound<'_, PyAny>,
) -> PyResult<String> {
let settings = RenderSettings {
set_difference_operator: adapter
.call_method0("render_set_difference_operator")?
.extract()?,
sql_analysis_dialect: adapter.call_method0("sql_analysis_dialect")?.extract()?,
probe_step_index: None,
output_step_index: None,
actual_relations: HashMap::new(),
};
rendered_comparisons(entry.py(), std::slice::from_ref(entry), &settings)?
.into_iter()
.next()
.ok_or_else(|| value_error("native SQL-test rendering returned an invalid batch response"))
}

/// The native comparison render request of one planned SQL test.
pub(crate) fn comparison_request(
entry: &Bound<'_, PyAny>,
Expand Down Expand Up @@ -183,7 +163,7 @@ pub(crate) fn chain_step_request(step: &Bound<'_, PyAny>) -> PyResult<Value> {
/// A bounded query for one direction of an expected-output difference of one chain step.
#[pyfunction]
#[pyo3(signature = (**options))]
fn render_sql_test_difference_sample_sql(
pub(crate) fn render_sql_test_difference_sample_sql(
py: Python<'_>,
options: Option<&Bound<'_, PyDict>>,
) -> PyResult<String> {
Expand Down Expand Up @@ -251,17 +231,3 @@ pub(crate) fn build_scenario_expected_comparison_sql(
set_difference_operator,
)
}

pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
module.add_function(wrap_pyfunction!(render_sql_test_comparison_sql, module)?)?;
module.add_function(wrap_pyfunction!(
render_sql_test_difference_sample_sql,
module
)?)?;
module.add_function(wrap_pyfunction!(describe_missing_expected_columns, module)?)?;
module.add_function(wrap_pyfunction!(
build_scenario_expected_comparison_sql,
module
)?)?;
Ok(())
}
15 changes: 15 additions & 0 deletions crates/sqlbuild-python-compile/src/analysis/_helpers/sql_tests.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
//! Extract, plan and render SQL tests natively.

use pyo3::prelude::{PyResult, Python};
use pyo3::pyfunction;

use sqlbuild_python_base::boundary::main::value_error::value_error;
use sqlbuild_python_base::types::CompilerDetach;

#[pyfunction]
pub(crate) fn extract_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult<String> {
py.compiler_detach(|| {
sqlbuild_analysis::compiler::main::sql_test_extraction::extract_batch_json(request_json)
})
.map_err(value_error)
}
12 changes: 12 additions & 0 deletions crates/sqlbuild-python-compile/src/analysis/constants.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
//! Public constants of analysis.

use crate::analysis::_helpers::getters::read_binding_catalog;
use sqlbuild_python_base::records::main::registry::class_of::class_of;
use sqlbuild_python_base::records::models::GetterEntry;
use sqlbuild_python_project::analysis::models::BindingCatalog;

/// The getter readers of this module's native pyclasses.
pub const GETTERS: &[GetterEntry] = &[GetterEntry {
class: class_of::<BindingCatalog>,
read: read_binding_catalog,
}];
Loading
Loading