Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
26 changes: 24 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ members = [
"crates/sqlbuild-python-base",
"crates/sqlbuild-python-front",
"crates/sqlbuild-python-project",
"crates/sqlbuild-python-planner",
"crates/sqlbuild-python",
]
resolver = "2"
Expand Down Expand Up @@ -61,9 +62,10 @@ order = [
"sqlbuild-python-base",
"sqlbuild-python-front",
"sqlbuild-python-project",
"sqlbuild-python-planner",
"sqlbuild-python",
]
python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python"]
python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python-planner", "sqlbuild-python"]
polyglot-floor = "sqlbuild-analysis"

[workspace.package]
Expand Down Expand Up @@ -186,6 +188,7 @@ sqlbuild-project = { path = "crates/sqlbuild-project" }
sqlbuild-python-base = { path = "crates/sqlbuild-python-base" }
sqlbuild-python-front = { path = "crates/sqlbuild-python-front" }
sqlbuild-python-project = { path = "crates/sqlbuild-python-project" }
sqlbuild-python-planner = { path = "crates/sqlbuild-python-planner" }
sqlbuild-pysource = { path = "crates/sqlbuild-pysource" }
sqlbuild-recall = { path = "crates/sqlbuild-recall" }
sqlbuild-scopes = { path = "crates/sqlbuild-scopes" }
Expand Down
5 changes: 3 additions & 2 deletions crates/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,15 @@ tested and benchmarked with `cargo test` alone.
| `sqlbuild-python-base` | The base of the PyO3 bindings: native record, enum and error classes with their specs, registration, construction and field reads; keyword calls, imports, conversions and panics at the Python boundary; host entry calls; the process allocator; and the shared runtime the binding modules build on: the native store, compile context, diagnostics, progress and terminal text, warehouse access and orchestration reads. |
| `sqlbuild-python-front` | The front of the compile pipeline as PyO3 bindings, on top of `sqlbuild-python-base`: project discovery, model configuration and headers, scopes, macro rendering and authored values, SQL text, model attachment and the compile driver, cursors, contract and refactor facts, and the warehouse state tables of consumption, observations and audit results. |
| `sqlbuild-python-project` | The compiled project as PyO3 bindings, on top of `sqlbuild-python-front`: the project host and the Python views of compiled models, inputs and resources, the compiled-project snapshot, analysis classes and the analysis session, the project graph, references, lineage, fingerprints, contracts and semantic shapes, audit rendering, and the planner's identities, scope, selection, phases, fixture strategy and SQL test planning. |
| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. |
| `sqlbuild-python-planner` | The plan layer as PyO3 bindings, on top of `sqlbuild-python-project`: execution plans and their entries, planner phases, change detection, migrations, reference and cursor resolution, scenarios, strategy and warehouse inspection, plus freshness, consumption, microbatches, observations, lifecycle events and plan output. |
| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-planner`, `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. |

## Layering

Dependencies point one way, from the top of this graph to the bottom:

```text
sqlbuild-python -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base
sqlbuild-python -> sqlbuild-python-planner -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base
-> sqlbuild-project -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments
-> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host
-> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core
Expand Down
48 changes: 48 additions & 0 deletions crates/sqlbuild-python-front/src/cost_context/_helpers/context.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
//! The `ContextVar` that holds the current cost resource context, and the scopes built over it.

use crate::cost_context::main::process_object::process_object;
use crate::cost_context::models::CostScope;
use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python};
use pyo3::sync::PyOnceLock;
use pyo3::types::PyDict;

use sqlbuild_python_base::boundary::main::imported::imported;
use sqlbuild_python_base::records::main::fields::field::field as native_field;

const CONTEXT_VARIABLE_NAME: &str = "sqlbuild_cost_resource_context";
pub(crate) const DEFAULT_PHASE: &str = "execute";
pub(crate) const DEFAULT_ATTEMPT: i64 = 1;

static CURRENT: PyOnceLock<Py<PyAny>> = PyOnceLock::new();

/// The `ContextVar` holding the current cost resource context.
pub(crate) fn current_variable(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> {
process_object(&CURRENT, py, || {
let options: Bound<'_, PyDict> = PyDict::new(py);
options.set_item("default", py.None())?;
imported(py, "contextvars", "ContextVar")?.call((CONTEXT_VARIABLE_NAME,), Some(&options))
})
}

pub(crate) fn or_current<'py>(
value: Bound<'py, PyAny>,
current: &Bound<'py, PyAny>,
field: &str,
) -> PyResult<Bound<'py, PyAny>> {
if !value.is_none() || current.is_none() {
return Ok(value);
}
native_field(current, field)
}

pub(crate) fn scope_of(context: Bound<'_, PyAny>) -> CostScope {
CostScope {
context: Some(context.unbind()),
token: None,
}
}

/// The active `CostResourceContext`, or `None`.
pub(crate) fn current_context(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
current_variable(py)?.call_method0("get")
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pub(crate) mod context;
6 changes: 6 additions & 0 deletions crates/sqlbuild-python-front/src/cost_context/constants.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
//! Constants of cost context.

/// The methods an adapter with warehouse cost telemetry implements.
pub(crate) const COST_AWARE_METHODS: [&str; 2] = ["cost_capability", "collect_run_cost"];

pub(crate) const DECIMAL_MODULE: &str = "decimal";
15 changes: 15 additions & 0 deletions crates/sqlbuild-python-front/src/cost_context/main/cost_aware.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
//! Whether an adapter implements the cost telemetry methods.

use crate::cost_context::constants::COST_AWARE_METHODS;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
use sqlbuild_python_base::records::main::fields::field::field as native_field;

/// Whether `adapter` implements the cost telemetry methods.
pub fn cost_aware(adapter: &Bound<'_, PyAny>) -> PyResult<bool> {
for name in COST_AWARE_METHODS {
if !adapter.hasattr(name)? || native_field(adapter, name)?.is_none() {
return Ok(false);
}
}
Ok(true)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
//! Microseconds since the epoch: UTC for aware values, the wall clock for naive ones.

use crate::cursor::constants::DATETIME_MODULE;
use crate::state_rows::main::event_rows::offset_aware::offset_aware;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python};
use pyo3::types::PyDict;
use sqlbuild_python_base::boundary::main::imported::imported;

/// Microseconds since the epoch: UTC for aware values, the wall clock for naive ones.
pub fn instant_micros(value: &Bound<'_, PyAny>) -> PyResult<i128> {
let py: Python<'_> = value.py();
let datetime: Bound<'_, PyAny> = imported(py, DATETIME_MODULE, "datetime")?;
let epoch: Bound<'_, PyAny> = if offset_aware(value)? {
let options: Bound<'_, PyDict> = PyDict::new(py);
options.set_item("tzinfo", imported(py, DATETIME_MODULE, "UTC")?)?;
datetime.call((1970, 1, 1), Some(&options))?
} else {
datetime.call1((1970, 1, 1))?
};
let options: Bound<'_, PyDict> = PyDict::new(py);
options.set_item("microseconds", 1)?;
let micro: Bound<'_, PyAny> =
imported(py, DATETIME_MODULE, "timedelta")?.call((), Some(&options))?;
value.sub(epoch)?.floor_div(micro)?.extract()
}
6 changes: 6 additions & 0 deletions crates/sqlbuild-python-front/src/cost_context/main/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pub mod cost_aware;
pub mod instant_micros;
pub mod native_decimal;
pub mod process_object;
pub mod python_decimal;
pub mod register_cost_context;
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
//! A finite non-negative `Decimal` as its exact digits and exponent.

use pyo3::exceptions::PyValueError;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
use sqlbuild_core::cost_allocation::models::PyDecimal;

/// A finite non-negative `Decimal` as its exact digits and exponent.
pub fn native_decimal(value: &Bound<'_, PyAny>) -> PyResult<PyDecimal> {
let (sign, digits, exponent): (u8, Vec<u8>, Bound<'_, PyAny>) =
value.call_method0("as_tuple")?.extract()?;
let Ok(exponent) = exponent.extract::<i64>() else {
return Err(PyValueError::new_err("cost decimals must be finite"));
};
if sign != 0 && digits.iter().any(|digit| *digit != 0) {
return Err(PyValueError::new_err("cost decimals must not be negative"));
}
let leading: usize = digits.iter().take_while(|digit| **digit == 0).count();
let digits: Vec<u8> = if leading == digits.len() {
vec![0]
} else {
digits[leading..].to_vec()
};
Ok(PyDecimal { digits, exponent })
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
//! The process-wide Python object in a cell, built on first use.

use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyResult, Python};
use pyo3::sync::PyOnceLock;

/// The process-wide Python object in `cell`, built by `build` on first use.
pub fn process_object<'py>(
cell: &'static PyOnceLock<Py<PyAny>>,
py: Python<'py>,
build: impl FnOnce() -> PyResult<Bound<'py, PyAny>>,
) -> PyResult<&'py Bound<'py, PyAny>> {
cell.get_or_try_init(py, || Ok::<Py<PyAny>, PyErr>(build()?.unbind()))
.map(|object| object.bind(py))
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
//! `Decimal((0, digits, exponent))`.

use crate::cost_context::constants::DECIMAL_MODULE;
use pyo3::IntoPyObjectExt;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python};
use pyo3::types::PyTuple;
use sqlbuild_core::cost_allocation::models::PyDecimal;
use sqlbuild_python_base::boundary::main::imported::imported;

/// `Decimal((0, digits, exponent))`.
pub fn python_decimal<'py>(py: Python<'py>, value: &PyDecimal) -> PyResult<Bound<'py, PyAny>> {
let parts: Bound<'py, PyTuple> = PyTuple::new(
py,
[
0_i64.into_bound_py_any(py)?,
PyTuple::new(py, &value.digits)?.into_any(),
value.exponent.into_bound_py_any(py)?,
],
)?;
imported(py, DECIMAL_MODULE, "Decimal")?.call1((parts,))
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
//! Register the cost context classes.

use crate::cost_context::models::{CostContext, CostScope};
use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult};

/// Register the cost context classes.
pub fn register_cost_context(module: &Bound<'_, PyModule>) -> PyResult<()> {
module.add_class::<CostContext>()?;
module.add_class::<CostScope>()?;
Ok(())
}
6 changes: 6 additions & 0 deletions crates/sqlbuild-python-front/src/cost_context/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
//! The per-resource cost attribution context and the Python cost values it reads and builds.

pub(crate) mod _helpers;
pub(crate) mod constants;
pub mod main;
pub mod models;
Original file line number Diff line number Diff line change
@@ -1,42 +1,23 @@
//! `CostContext`: the per-resource cost attribution context, held in a Python `ContextVar`.
//! Public records of cost context.

use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python};
use pyo3::sync::PyOnceLock;
use crate::cost_context::_helpers::context::{
DEFAULT_ATTEMPT, DEFAULT_PHASE, current_context, current_variable, or_current, scope_of,
};
use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python};
use pyo3::types::{PyDict, PyTuple};
use pyo3::{IntoPyObjectExt, pyclass, pymethods};
use sqlbuild_host::entries::models::ExecutorHost;

use crate::bindings::_helpers::cost_telemetry::values::process_object;
use sqlbuild_python_base::boundary::main::call_with::call_with;
use sqlbuild_python_base::boundary::main::imported::imported;
use sqlbuild_python_base::boundary::models::KeywordArguments;
use sqlbuild_python_base::native_host::main::host_entry::host_entry;
use sqlbuild_python_base::records::main::fields::field::field as native_field;

const CONTEXT_VARIABLE_NAME: &str = "sqlbuild_cost_resource_context";
const DEFAULT_PHASE: &str = "execute";
const DEFAULT_ATTEMPT: i64 = 1;

static CURRENT: PyOnceLock<Py<PyAny>> = PyOnceLock::new();

fn current_variable(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> {
process_object(&CURRENT, py, || {
let options: Bound<'_, PyDict> = PyDict::new(py);
options.set_item("default", py.None())?;
imported(py, "contextvars", "ContextVar")?.call((CONTEXT_VARIABLE_NAME,), Some(&options))
})
}

/// The active `CostResourceContext`, or `None`.
pub(crate) fn current_context(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
current_variable(py)?.call_method0("get")
}

/// A `with` scope that makes one resource context current, or does nothing without one.
#[pyclass(module = "sqlbuild._native")]
pub(crate) struct CostScope {
context: Option<Py<PyAny>>,
token: Option<Py<PyAny>>,
#[derive(Debug)]
pub struct CostScope {
pub(crate) context: Option<Py<PyAny>>,
pub(crate) token: Option<Py<PyAny>>,
}

#[pymethods]
Expand All @@ -61,27 +42,10 @@ impl CostScope {
}
}

fn or_current<'py>(
value: Bound<'py, PyAny>,
current: &Bound<'py, PyAny>,
field: &str,
) -> PyResult<Bound<'py, PyAny>> {
if !value.is_none() || current.is_none() {
return Ok(value);
}
native_field(current, field)
}

fn scope_of(context: Bound<'_, PyAny>) -> CostScope {
CostScope {
context: Some(context.unbind()),
token: None,
}
}

/// Run-cost execution context entry points.
#[pyclass(module = "sqlbuild._native", frozen)]
pub(crate) struct CostContext;
#[derive(Debug)]
pub struct CostContext;

#[pymethods]
impl CostContext {
Expand All @@ -92,7 +56,7 @@ impl CostContext {

#[staticmethod]
#[pyo3(signature = (**options))]
pub(crate) fn scope(py: Python<'_>, options: Option<Bound<'_, PyDict>>) -> PyResult<CostScope> {
pub fn scope(py: Python<'_>, options: Option<Bound<'_, PyDict>>) -> PyResult<CostScope> {
let arguments: KeywordArguments<'_> = KeywordArguments::read(
py,
"scope",
Expand Down Expand Up @@ -139,7 +103,7 @@ impl CostContext {

#[staticmethod]
#[pyo3(signature = (**options))]
pub(crate) fn resource_scope(
pub fn resource_scope(
py: Python<'_>,
options: Option<Bound<'_, PyDict>>,
) -> PyResult<CostScope> {
Expand Down Expand Up @@ -187,9 +151,3 @@ impl CostContext {
Ok(scope_of(context))
}
}

pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
module.add_class::<CostContext>()?;
module.add_class::<CostScope>()?;
Ok(())
}
Loading
Loading