Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ tested and benchmarked with `cargo test` alone.
| `sqlbuild-project` | The compiled project as plain Rust: `CompiledProject` with the producers' facts and each resource's records (model inputs and schemas, compiled models and their analysis, sources, functions, audits and seeds), and what those records answer without Python, such as lineage rows, resolved namespaces and dependency keys. Values the host owns, such as user config and discovered files, and the views it builds lazily live in one slot per record, whose types `sqlbuild-python-project` supplies through the `ProjectHost` trait; reads that need Python stay in the bindings as extension traits. |
| `sqlbuild-cli` | The `sqb` and `sqlbuild` executables: `--version`, unchanged compiles replayed from the shared reuse store with Python's exact streams, exit code and compute log capture, and a handoff of every other invocation to the installed `sqlbuild-python` script. Built as its own wheel (`crates/sqlbuild-cli/pyproject.toml`). |
| `sqlbuild-python-base` | The base of the PyO3 bindings: native record, enum and error classes with their specs, registration, construction and field reads; keyword calls, imports, conversions and panics at the Python boundary; host entry calls; the process allocator; and the shared runtime the binding modules build on: the native store, compile context, diagnostics, progress and terminal text, warehouse access and orchestration reads. |
| `sqlbuild-python-front` | The front of the compile pipeline as PyO3 bindings, on top of `sqlbuild-python-base`: project discovery, model configuration and headers, scopes, macro rendering and authored values, SQL text, model attachment and the compile driver, cursors, contract and refactor facts, and the warehouse state tables of consumption, observations and audit results. |
| `sqlbuild-python-front` | The front of the compile pipeline as PyO3 bindings, on top of `sqlbuild-python-base`: project discovery, model configuration and headers, scopes, macro rendering and authored values, SQL text, model attachment and the compile driver, cursors, contract and refactor facts, the warehouse state tables of consumption, observations and audit results, the execution history store and the resource costs table. |
| `sqlbuild-python-project` | The compiled project as PyO3 bindings, on top of `sqlbuild-python-front`: the project host and the Python views of compiled models, inputs and resources, the compiled-project snapshot, analysis classes and the analysis session, the project graph, references, lineage, fingerprints, contracts and semantic shapes, audit rendering, and the planner's identities, scope, selection, phases, fixture strategy and SQL test planning. |
| `sqlbuild-python-planner` | The plan layer as PyO3 bindings, on top of `sqlbuild-python-project`: execution plans and their entries, planner phases, change detection, migrations, reference and cursor resolution, scenarios, strategy and warehouse inspection, plus freshness, consumption, microbatches, observations, lifecycle events and plan output. |
| `sqlbuild-python-compile` | The compile layer as PyO3 bindings, on top of `sqlbuild-python-project` and beside `sqlbuild-python-planner`: the compile command and driver, input loops and compile-input attachment, project assembly, analysis and the analysis session, Rules, lineage, semantic checks, compile deltas and outputs, the DAG artifact, manifest, frontier, offline scope facts, SQL test glue and auditing. |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@ use sqlbuild_state::execution_history::main::canonical_event_id::canonical_event
use sqlbuild_state::execution_history::main::prepared_events::prepared_events;
use sqlbuild_state::execution_history::models::{EventSubmission, PreparedEvent};

use crate::bindings::_helpers::execution_history::conversion::{
checked, envelope_of, event_id_of, event_json,
};
use crate::execution_history::_helpers::conversion::checked;
use crate::execution_history::_helpers::conversion::envelope_of;
use crate::execution_history::_helpers::conversion::event_id_of;
use crate::execution_history::_helpers::conversion::event_json;

/// The required stable id of a known or opaque event.
pub(crate) fn event_id(event: &Bound<'_, PyAny>) -> PyResult<String> {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
//! Execution history models, timestamps, SQL parameters and errors across the Python boundary.

use crate::execution_history::main::nullable_text::nullable_text;
use crate::execution_history::main::parsed_iso::parsed_iso;
use crate::execution_history::main::utc::utc;
use pyo3::IntoPyObjectExt;
use pyo3::exceptions::PyValueError;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python};
use pyo3::types::{PyBool, PyInt, PyList, PyTuple};
use sqlbuild_host::entries::models::ExecutorHost;
Expand All @@ -15,15 +17,15 @@ use sqlbuild_state::execution_history::models::{
};
use sqlbuild_state::execution_history::types::UtcTime;

use crate::cursor::constants::DATETIME_MODULE;
use crate::cursor::main::values::local_date_time::local_date_time;
use sqlbuild_python_base::boundary::main::call_with::call_with;
use sqlbuild_python_base::boundary::main::imported::imported;
use sqlbuild_python_base::native_host::main::host_entry::host_entry;
use sqlbuild_python_base::records::main::fields::field::field as native_field;
use sqlbuild_python_front::cursor::constants::DATETIME_MODULE;
use sqlbuild_python_front::cursor::main::values::local_date_time::local_date_time;

const UTC_SUFFIX: &str = "Z";
const UTC_OFFSET: &str = "+00:00";
pub(crate) const UTC_SUFFIX: &str = "Z";
pub(crate) const UTC_OFFSET: &str = "+00:00";
const DEFAULT_SCHEMA_VERSION: i64 = 1;
const UTC_OFFSET_SECONDS: f64 = 0.0;

Expand Down Expand Up @@ -79,19 +81,6 @@ pub(crate) fn exact_int(value: &Bound<'_, PyAny>) -> Option<i64> {
value.extract().unwrap_or(None)
}

/// `value` as text, or `None` when it is `None`.
pub(crate) fn nullable_text(value: &Bound<'_, PyAny>) -> PyResult<Option<String>> {
if value.is_none() {
return Ok(None);
}
Ok(Some(value.extract()?))
}

/// `datetime.UTC`.
pub(crate) fn utc(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
imported(py, DATETIME_MODULE, "UTC")
}

/// An aware `datetime` as its UTC wall time.
pub(crate) fn utc_time(value: &Bound<'_, PyAny>) -> PyResult<UtcTime> {
local_date_time(&value.call_method1("astimezone", (utc(value.py())?,))?)
Expand Down Expand Up @@ -122,16 +111,6 @@ pub(crate) fn utc_zone(value: &Bound<'_, PyAny>) -> PyResult<bool> {
Ok(native_field(value, "tzinfo")?.is(&utc(value.py())?))
}

/// `datetime.fromisoformat(text.replace("Z", "+00:00"))`, or `None` on `ValueError`.
pub(crate) fn parsed_iso<'py>(py: Python<'py>, text: &str) -> PyResult<Option<Bound<'py, PyAny>>> {
let normalized: String = text.replace(UTC_SUFFIX, UTC_OFFSET);
match imported(py, DATETIME_MODULE, "datetime")?.call_method1("fromisoformat", (normalized,)) {
Ok(value) => Ok(Some(value)),
Err(error) if error.is_instance_of::<PyValueError>(py) => Ok(None),
Err(error) => Err(error),
}
}

fn filter_time(value: &Bound<'_, PyAny>) -> PyResult<Option<FilterTime>> {
if value.is_none() {
return Ok(None);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
//! Backend-neutral execution history entry points and model validation.
use pyo3::prelude::{
Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, pyfunction,
wrap_pyfunction,
};
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction};
use pyo3::types::PyTuple;
use sqlbuild_python_base::boundary::main::optional_text::optional_text;
use sqlbuild_state::execution_history::main::project_runs::project_runs;
Expand All @@ -13,29 +10,34 @@ use sqlbuild_state::execution_history::main::validate_page_limit::validate_page_
use sqlbuild_state::execution_history::main::validate_stored_event::validate_stored_event;
use sqlbuild_state::execution_history::models::FilterSpec;

use crate::bindings::_helpers::execution_history::appends::{event_content, event_id};
use crate::bindings::_helpers::execution_history::conversion::{
checked, event_filter_spec, exact_int, projection_facts, run_filter_spec, run_records,
run_rows, utc_zone,
};
use crate::execution_history::_helpers::appends::event_content;
use crate::execution_history::_helpers::appends::event_id;
use crate::execution_history::_helpers::conversion::checked;
use crate::execution_history::_helpers::conversion::event_filter_spec;
use crate::execution_history::_helpers::conversion::exact_int;
use crate::execution_history::_helpers::conversion::projection_facts;
use crate::execution_history::_helpers::conversion::run_filter_spec;
use crate::execution_history::_helpers::conversion::run_records;
use crate::execution_history::_helpers::conversion::run_rows;
use crate::execution_history::_helpers::conversion::utc_zone;
use sqlbuild_python_base::records::main::fields::field::field as native_field;

/// Return the required stable ID from a known or opaque lifecycle event.
#[pyfunction]
fn canonical_event_id(event: &Bound<'_, PyAny>) -> PyResult<String> {
pub(crate) fn canonical_event_id(event: &Bound<'_, PyAny>) -> PyResult<String> {
event_id(event)
}

/// Return deterministic content for lifecycle event idempotency comparison.
#[pyfunction]
fn canonical_event_content(event: &Bound<'_, PyAny>) -> PyResult<String> {
pub(crate) fn canonical_event_content(event: &Bound<'_, PyAny>) -> PyResult<String> {
event_content(event)
}

/// Apply durable event facts in storage order to immutable run projections.
#[pyfunction]
#[pyo3(signature = (*, stored_events, current_runs = None))]
fn project_execution_runs<'py>(
pub(crate) fn project_execution_runs<'py>(
py: Python<'py>,
stored_events: &Bound<'py, PyAny>,
current_runs: Option<&Bound<'py, PyAny>>,
Expand All @@ -50,14 +52,14 @@ fn project_execution_runs<'py>(

/// Validate a positive page limit no greater than the public maximum.
#[pyfunction]
fn validate_history_page_limit(limit: &Bound<'_, PyAny>) -> Result<(), PyErr> {
pub(crate) fn validate_history_page_limit(limit: &Bound<'_, PyAny>) -> Result<(), PyErr> {
checked(limit.py(), validate_page_limit(exact_int(limit)))
}

/// Append facts durably before publishing their rebuildable run projection.
#[pyfunction]
#[pyo3(signature = (*, event_log, run_storage, events))]
fn append_and_project<'py>(
pub(crate) fn append_and_project<'py>(
event_log: &Bound<'py, PyAny>,
run_storage: &Bound<'py, PyAny>,
events: &Bound<'py, PyAny>,
Expand All @@ -69,7 +71,7 @@ fn append_and_project<'py>(

/// Validate a `StoredEvent`'s position, cursor and UTC receipt time.
#[pyfunction]
fn validate_stored_history_event(record: &Bound<'_, PyAny>) -> Result<(), PyErr> {
pub(crate) fn validate_stored_history_event(record: &Bound<'_, PyAny>) -> Result<(), PyErr> {
let py: Python<'_> = record.py();
let storage_order: i64 = native_field(record, "storage_order")?.extract()?;
let cursor: String = optional_text(&native_field(record, "cursor")?)?.unwrap_or_default();
Expand All @@ -82,7 +84,7 @@ fn validate_stored_history_event(record: &Bound<'_, PyAny>) -> Result<(), PyErr>

/// Validate an `EventFilter`.
#[pyfunction]
fn validate_history_event_filter(event_filter: &Bound<'_, PyAny>) -> Result<(), PyErr> {
pub(crate) fn validate_history_event_filter(event_filter: &Bound<'_, PyAny>) -> Result<(), PyErr> {
checked(
event_filter.py(),
validate_filter(FilterSpec::Event(&event_filter_spec(event_filter)?)),
Expand All @@ -91,7 +93,7 @@ fn validate_history_event_filter(event_filter: &Bound<'_, PyAny>) -> Result<(),

/// Validate a `RunFilter`.
#[pyfunction]
fn validate_history_run_filter(run_filter: &Bound<'_, PyAny>) -> Result<(), PyErr> {
pub(crate) fn validate_history_run_filter(run_filter: &Bound<'_, PyAny>) -> Result<(), PyErr> {
checked(
run_filter.py(),
validate_filter(FilterSpec::Run(&run_filter_spec(run_filter)?)),
Expand All @@ -101,7 +103,10 @@ fn validate_history_run_filter(run_filter: &Bound<'_, PyAny>) -> Result<(), PyEr
/// Validate an `EventPage` or `RunPage` next cursor against its records.
#[pyfunction]
#[pyo3(signature = (*, page, is_event_page))]
fn validate_history_page(page: &Bound<'_, PyAny>, is_event_page: bool) -> Result<(), PyErr> {
pub(crate) fn validate_history_page(
page: &Bound<'_, PyAny>,
is_event_page: bool,
) -> Result<(), PyErr> {
let py: Python<'_> = page.py();
let records: Bound<'_, PyAny> = native_field(page, "records")?;
let has_records: bool = records.is_truthy()?;
Expand All @@ -122,15 +127,3 @@ fn validate_history_page(page: &Bound<'_, PyAny>, is_event_page: bool) -> Result
validate_page(is_event_page, has_records, (next, last_text.as_deref())),
)
}

pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
module.add_function(wrap_pyfunction!(canonical_event_id, module)?)?;
module.add_function(wrap_pyfunction!(canonical_event_content, module)?)?;
module.add_function(wrap_pyfunction!(project_execution_runs, module)?)?;
module.add_function(wrap_pyfunction!(validate_history_page_limit, module)?)?;
module.add_function(wrap_pyfunction!(append_and_project, module)?)?;
module.add_function(wrap_pyfunction!(validate_stored_history_event, module)?)?;
module.add_function(wrap_pyfunction!(validate_history_event_filter, module)?)?;
module.add_function(wrap_pyfunction!(validate_history_run_filter, module)?)?;
module.add_function(wrap_pyfunction!(validate_history_page, module)?)
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,24 +3,10 @@
use pyo3::IntoPyObjectExt;
use pyo3::prelude::{Bound, PyAny, PyResult, Python};

use crate::bindings::_helpers::execution_history::postgres::PostgresExecutionHistory;
use crate::bindings::_helpers::execution_history::sqlite::SqliteExecutionHistory;
use sqlbuild_python_base::records::main::registry::class_of::class_of;
use sqlbuild_python_base::records::models::GetterEntry;
use crate::execution_history::models::PostgresExecutionHistory;
use crate::execution_history::models::SqliteExecutionHistory;

/// The getter readers of this module's native pyclasses.
pub(crate) const GETTERS: &[GetterEntry] = &[
GetterEntry {
class: class_of::<PostgresExecutionHistory>,
read: read_postgres_execution_history,
},
GetterEntry {
class: class_of::<SqliteExecutionHistory>,
read: read_sqlite_execution_history,
},
];

fn read_postgres_execution_history<'py>(
pub(crate) fn read_postgres_execution_history<'py>(
value: &Bound<'py, PyAny>,
name: &str,
) -> Option<PyResult<Bound<'py, PyAny>>> {
Expand All @@ -37,7 +23,7 @@ fn read_postgres_execution_history<'py>(
})
}

fn read_sqlite_execution_history<'py>(
pub(crate) fn read_sqlite_execution_history<'py>(
value: &Bound<'py, PyAny>,
name: &str,
) -> Option<PyResult<Bound<'py, PyAny>>> {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
pub(crate) mod appends;
pub(crate) mod conversion;
pub(crate) mod functions;
pub(crate) mod getters;
pub(crate) mod postgres;
pub(crate) mod sqlite;
pub(crate) mod statements;
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
//! `PostgresExecutionHistory`: deployed PostgreSQL event-log and run-projection storage.

use crate::execution_history::main::utc::utc;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python};
use pyo3::types::PyDateTime;
use sqlbuild_host::entries::models::ExecutorHost;
use sqlbuild_state::execution_history::constants::POSTGRES_RETRYABLE_SQLSTATES;
use sqlbuild_state::execution_history::models::RunRow;
use sqlbuild_state::execution_history::types::UtcTime;

use crate::execution_history::_helpers::conversion::exact_int;
use crate::execution_history::_helpers::conversion::parsed_status;
use crate::execution_history::_helpers::conversion::storage_error;
use crate::execution_history::_helpers::conversion::utc_time;
use crate::execution_history::_helpers::statements::column;
use sqlbuild_python_base::boundary::main::imported::imported;

pub(crate) const PSYCOPG_MODULE: &str = "psycopg";
pub(crate) const PSYCOPG_ROWS_MODULE: &str = "psycopg.rows";
pub(crate) const CLOSED: &str = "PostgreSQL execution history storage is closed";
pub(crate) const MISSING_DRIVER: &str = "PostgreSQL execution history requires psycopg 3; install sqlbuild[postgres] or 'psycopg[binary]>=3.2'";
pub(crate) const CONNECT_FAILED: &str = "cannot connect to PostgreSQL execution history; verify the secret-resolved DSN, network access, and sqlbuild[postgres] installation";
pub(crate) const READ_FAILED: &str = "PostgreSQL execution history read failed";
pub(crate) const METADATA_INVALID: &str = "PostgreSQL execution history schema metadata is invalid";
pub(crate) const NAMESPACE_MISSING: &str =
"PostgreSQL execution history cursor namespace is missing";
pub(crate) const STORAGE_ERROR: ExecutorHost = ExecutorHost::ExecutionHistoryStorageError;

/// The `psycopg.Error` class.
fn driver_error_class(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
imported(py, PSYCOPG_MODULE, "Error")
}

pub(crate) fn driver_failure(py: Python<'_>, error: &PyErr) -> PyResult<bool> {
Ok(error.is_instance(py, &driver_error_class(py)?))
}

/// Serialization failures and deadlocks retry while attempts remain.
pub(crate) fn retries_transaction(sqlstate: Option<&str>, attempts: i64, max_retries: i64) -> bool {
sqlstate.is_some_and(|state| POSTGRES_RETRYABLE_SQLSTATES.contains(&state))
&& attempts < max_retries
}

/// `value` when it is an `int` that is not a `bool`.
pub(crate) fn storage_position(value: &Bound<'_, PyAny>, message: &str) -> PyResult<i64> {
exact_int(value).ok_or_else(|| storage_error(value.py(), message))
}

/// `value.astimezone(UTC)` when it is a `datetime`.
pub(crate) fn utc_datetime<'py>(
value: &Bound<'py, PyAny>,
message: &str,
) -> PyResult<Bound<'py, PyAny>> {
let py: Python<'py> = value.py();
if !value.is_instance_of::<PyDateTime>() {
return Err(storage_error(py, message));
}
value.call_method1("astimezone", (utc(py)?,))
}

fn optional_utc(value: &Bound<'_, PyAny>) -> PyResult<Option<UtcTime>> {
if !value.is_instance_of::<PyDateTime>() {
return Ok(None);
}
utc_time(value).map(Some)
}

fn text_column(row: &Bound<'_, PyAny>, name: &str) -> PyResult<String> {
column(row, name)?.str()?.extract()
}

/// One `sqlbuild_run_projection` row as a projected run.
pub(crate) fn stored_run(row: &Bound<'_, PyAny>) -> PyResult<RunRow> {
let py: Python<'_> = row.py();
let created_at = utc_datetime(
&column(row, "created_at")?,
"PostgreSQL run timestamp is invalid",
)?;
let last_storage_order: i64 = storage_position(
&column(row, "last_event_storage_id")?,
"PostgreSQL run storage position is invalid",
)?;
let optional_str = |name: &str| -> PyResult<Option<String>> {
let value = column(row, name)?;
if value.is_none() {
return Ok(None);
}
Ok(Some(value.str()?.extract()?))
};
Ok(RunRow {
run_id: text_column(row, "run_id")?,
invocation_id: text_column(row, "invocation_id")?,
created_at: utc_time(&created_at)?,
status: parsed_status(py, &text_column(row, "status")?)?,
is_complete: column(row, "is_complete")?.is_truthy()?,
last_event_cursor: text_column(row, "last_event_cursor")?,
last_storage_order,
command: optional_str("command")?,
target: optional_str("target")?,
environment: optional_str("environment")?,
started_at: optional_utc(&column(row, "started_at")?)?,
ended_at: optional_utc(&column(row, "ended_at")?)?,
})
}
Loading
Loading