From a3d98d42913006e610f38a9ecff93cbda4fb382f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 08:24:47 +0000 Subject: [PATCH 1/2] refactor: move the execution history store into sqlbuild-python-front The execution history store (its SQLite and PostgreSQL backends, appends, statements, conversions and Python functions) and the resource costs table move from sqlbuild-python into sqlbuild-python-front, the layer fensu already assigns them. Nothing in a lower crate imports them, so the cost report can follow later without reaching back into sqlbuild-python. Old paths are deleted and callers updated; there are no re-exports. The review command changes only its imports. main/ entries hold the bodies they used to forward to. The resource costs table stays an XSB072/XSB073 edge as it was under native_host. The sqlbuild-python cap drops to its new size, and the moved files' budgets are re-keyed with unchanged totals. --- crates/README.md | 2 +- .../execution_history/_helpers}/appends.rs | 7 +- .../execution_history/_helpers}/conversion.rs | 35 +- .../execution_history/_helpers}/functions.rs | 53 +- .../execution_history/_helpers}/getters.rs | 22 +- .../src/execution_history/_helpers/mod.rs | 7 + .../execution_history/_helpers/postgres.rs | 104 + .../src/execution_history/_helpers/sqlite.rs | 140 ++ .../execution_history/_helpers}/statements.rs | 16 +- .../src/execution_history/constants.rs | 20 + .../src/execution_history/main/fetch_one.rs | 17 + .../src/execution_history/main/mod.rs | 5 + .../execution_history/main/nullable_text.rs | 11 + .../src/execution_history/main/parsed_iso.rs | 18 + .../main/register_functions.rs | 25 + .../src/execution_history/main/utc.rs | 10 + .../src/execution_history/mod.rs | 6 + .../src/execution_history/models.rs | 1833 +++++++++++++++++ crates/sqlbuild-python-front/src/lib.rs | 2 + .../src/resource_costs/_helpers/mod.rs | 1 + .../_helpers}/resource_costs_state.rs | 208 +- .../src/resource_costs/main/mod.rs | 1 + .../main/register_resource_costs_state.rs | 15 + .../src/resource_costs/mod.rs | 5 + .../src/resource_costs/models.rs | 183 ++ .../src/bindings/_helpers/cli/registration.rs | 2 +- .../_helpers/cost_report/publication.rs | 5 +- .../_helpers/execution_history/mod.rs | 7 - .../_helpers/execution_history/postgres.rs | 993 --------- .../execution_history/registration.rs | 8 +- .../_helpers/execution_history/sqlite.rs | 945 --------- .../src/bindings/_helpers/native_host/mod.rs | 1 - .../_helpers/record_behaviours/constants.rs | 5 +- .../_helpers/review_command/comparison.rs | 4 +- .../_helpers/review_command/planning.rs | 2 +- .../_helpers/review_command/support.rs | 2 +- fensu.toml | 63 +- 37 files changed, 2527 insertions(+), 2256 deletions(-) rename crates/{sqlbuild-python/src/bindings/_helpers/execution_history => sqlbuild-python-front/src/execution_history/_helpers}/appends.rs (84%) rename crates/{sqlbuild-python/src/bindings/_helpers/execution_history => sqlbuild-python-front/src/execution_history/_helpers}/conversion.rs (93%) rename crates/{sqlbuild-python/src/bindings/_helpers/execution_history => sqlbuild-python-front/src/execution_history/_helpers}/functions.rs (67%) rename crates/{sqlbuild-python/src/bindings/_helpers/execution_history => sqlbuild-python-front/src/execution_history/_helpers}/getters.rs (60%) create mode 100644 crates/sqlbuild-python-front/src/execution_history/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-front/src/execution_history/_helpers/postgres.rs create mode 100644 crates/sqlbuild-python-front/src/execution_history/_helpers/sqlite.rs rename crates/{sqlbuild-python/src/bindings/_helpers/execution_history => sqlbuild-python-front/src/execution_history/_helpers}/statements.rs (77%) create mode 100644 crates/sqlbuild-python-front/src/execution_history/constants.rs create mode 100644 crates/sqlbuild-python-front/src/execution_history/main/fetch_one.rs create mode 100644 crates/sqlbuild-python-front/src/execution_history/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/execution_history/main/nullable_text.rs create mode 100644 crates/sqlbuild-python-front/src/execution_history/main/parsed_iso.rs create mode 100644 crates/sqlbuild-python-front/src/execution_history/main/register_functions.rs create mode 100644 crates/sqlbuild-python-front/src/execution_history/main/utc.rs create mode 100644 crates/sqlbuild-python-front/src/execution_history/mod.rs create mode 100644 crates/sqlbuild-python-front/src/execution_history/models.rs create mode 100644 crates/sqlbuild-python-front/src/resource_costs/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/native_host => sqlbuild-python-front/src/resource_costs/_helpers}/resource_costs_state.rs (53%) create mode 100644 crates/sqlbuild-python-front/src/resource_costs/main/mod.rs create mode 100644 crates/sqlbuild-python-front/src/resource_costs/main/register_resource_costs_state.rs create mode 100644 crates/sqlbuild-python-front/src/resource_costs/mod.rs create mode 100644 crates/sqlbuild-python-front/src/resource_costs/models.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs diff --git a/crates/README.md b/crates/README.md index a307c90e7e..1f76ae9b3c 100644 --- a/crates/README.md +++ b/crates/README.md @@ -29,7 +29,7 @@ tested and benchmarked with `cargo test` alone. | `sqlbuild-project` | The compiled project as plain Rust: `CompiledProject` with the producers' facts and each resource's records (model inputs and schemas, compiled models and their analysis, sources, functions, audits and seeds), and what those records answer without Python, such as lineage rows, resolved namespaces and dependency keys. Values the host owns, such as user config and discovered files, and the views it builds lazily live in one slot per record, whose types `sqlbuild-python-project` supplies through the `ProjectHost` trait; reads that need Python stay in the bindings as extension traits. | | `sqlbuild-cli` | The `sqb` and `sqlbuild` executables: `--version`, unchanged compiles replayed from the shared reuse store with Python's exact streams, exit code and compute log capture, and a handoff of every other invocation to the installed `sqlbuild-python` script. Built as its own wheel (`crates/sqlbuild-cli/pyproject.toml`). | | `sqlbuild-python-base` | The base of the PyO3 bindings: native record, enum and error classes with their specs, registration, construction and field reads; keyword calls, imports, conversions and panics at the Python boundary; host entry calls; the process allocator; and the shared runtime the binding modules build on: the native store, compile context, diagnostics, progress and terminal text, warehouse access and orchestration reads. | -| `sqlbuild-python-front` | The front of the compile pipeline as PyO3 bindings, on top of `sqlbuild-python-base`: project discovery, model configuration and headers, scopes, macro rendering and authored values, SQL text, model attachment and the compile driver, cursors, contract and refactor facts, and the warehouse state tables of consumption, observations and audit results. | +| `sqlbuild-python-front` | The front of the compile pipeline as PyO3 bindings, on top of `sqlbuild-python-base`: project discovery, model configuration and headers, scopes, macro rendering and authored values, SQL text, model attachment and the compile driver, cursors, contract and refactor facts, the warehouse state tables of consumption, observations and audit results, the execution history store and the resource costs table. | | `sqlbuild-python-project` | The compiled project as PyO3 bindings, on top of `sqlbuild-python-front`: the project host and the Python views of compiled models, inputs and resources, the compiled-project snapshot, analysis classes and the analysis session, the project graph, references, lineage, fingerprints, contracts and semantic shapes, audit rendering, and the planner's identities, scope, selection, phases, fixture strategy and SQL test planning. | | `sqlbuild-python-planner` | The plan layer as PyO3 bindings, on top of `sqlbuild-python-project`: execution plans and their entries, planner phases, change detection, migrations, reference and cursor resolution, scenarios, strategy and warehouse inspection, plus freshness, consumption, microbatches, observations, lifecycle events and plan output. | | `sqlbuild-python-compile` | The compile layer as PyO3 bindings, on top of `sqlbuild-python-project` and beside `sqlbuild-python-planner`: the compile command and driver, input loops and compile-input attachment, project assembly, analysis and the analysis session, Rules, lineage, semantic checks, compile deltas and outputs, the DAG artifact, manifest, frontier, offline scope facts, SQL test glue and auditing. | diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/appends.rs b/crates/sqlbuild-python-front/src/execution_history/_helpers/appends.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/execution_history/appends.rs rename to crates/sqlbuild-python-front/src/execution_history/_helpers/appends.rs index 1238a733fe..daa91d24f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/appends.rs +++ b/crates/sqlbuild-python-front/src/execution_history/_helpers/appends.rs @@ -5,9 +5,10 @@ use sqlbuild_state::execution_history::main::canonical_event_id::canonical_event use sqlbuild_state::execution_history::main::prepared_events::prepared_events; use sqlbuild_state::execution_history::models::{EventSubmission, PreparedEvent}; -use crate::bindings::_helpers::execution_history::conversion::{ - checked, envelope_of, event_id_of, event_json, -}; +use crate::execution_history::_helpers::conversion::checked; +use crate::execution_history::_helpers::conversion::envelope_of; +use crate::execution_history::_helpers::conversion::event_id_of; +use crate::execution_history::_helpers::conversion::event_json; /// The required stable id of a known or opaque event. pub(crate) fn event_id(event: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs b/crates/sqlbuild-python-front/src/execution_history/_helpers/conversion.rs similarity index 93% rename from crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs rename to crates/sqlbuild-python-front/src/execution_history/_helpers/conversion.rs index b86223f567..87ac6ddbd4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs +++ b/crates/sqlbuild-python-front/src/execution_history/_helpers/conversion.rs @@ -1,7 +1,9 @@ //! Execution history models, timestamps, SQL parameters and errors across the Python boundary. +use crate::execution_history::main::nullable_text::nullable_text; +use crate::execution_history::main::parsed_iso::parsed_iso; +use crate::execution_history::main::utc::utc; use pyo3::IntoPyObjectExt; -use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyInt, PyList, PyTuple}; use sqlbuild_host::entries::models::ExecutorHost; @@ -15,15 +17,15 @@ use sqlbuild_state::execution_history::models::{ }; use sqlbuild_state::execution_history::types::UtcTime; +use crate::cursor::constants::DATETIME_MODULE; +use crate::cursor::main::values::local_date_time::local_date_time; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::cursor::constants::DATETIME_MODULE; -use sqlbuild_python_front::cursor::main::values::local_date_time::local_date_time; -const UTC_SUFFIX: &str = "Z"; -const UTC_OFFSET: &str = "+00:00"; +pub(crate) const UTC_SUFFIX: &str = "Z"; +pub(crate) const UTC_OFFSET: &str = "+00:00"; const DEFAULT_SCHEMA_VERSION: i64 = 1; const UTC_OFFSET_SECONDS: f64 = 0.0; @@ -79,19 +81,6 @@ pub(crate) fn exact_int(value: &Bound<'_, PyAny>) -> Option { value.extract().unwrap_or(None) } -/// `value` as text, or `None` when it is `None`. -pub(crate) fn nullable_text(value: &Bound<'_, PyAny>) -> PyResult> { - if value.is_none() { - return Ok(None); - } - Ok(Some(value.extract()?)) -} - -/// `datetime.UTC`. -pub(crate) fn utc(py: Python<'_>) -> PyResult> { - imported(py, DATETIME_MODULE, "UTC") -} - /// An aware `datetime` as its UTC wall time. pub(crate) fn utc_time(value: &Bound<'_, PyAny>) -> PyResult { local_date_time(&value.call_method1("astimezone", (utc(value.py())?,))?) @@ -122,16 +111,6 @@ pub(crate) fn utc_zone(value: &Bound<'_, PyAny>) -> PyResult { Ok(native_field(value, "tzinfo")?.is(&utc(value.py())?)) } -/// `datetime.fromisoformat(text.replace("Z", "+00:00"))`, or `None` on `ValueError`. -pub(crate) fn parsed_iso<'py>(py: Python<'py>, text: &str) -> PyResult>> { - let normalized: String = text.replace(UTC_SUFFIX, UTC_OFFSET); - match imported(py, DATETIME_MODULE, "datetime")?.call_method1("fromisoformat", (normalized,)) { - Ok(value) => Ok(Some(value)), - Err(error) if error.is_instance_of::(py) => Ok(None), - Err(error) => Err(error), - } -} - fn filter_time(value: &Bound<'_, PyAny>) -> PyResult> { if value.is_none() { return Ok(None); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/functions.rs b/crates/sqlbuild-python-front/src/execution_history/_helpers/functions.rs similarity index 67% rename from crates/sqlbuild-python/src/bindings/_helpers/execution_history/functions.rs rename to crates/sqlbuild-python-front/src/execution_history/_helpers/functions.rs index 1a66d2882b..7cb3d9a3b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/functions.rs +++ b/crates/sqlbuild-python-front/src/execution_history/_helpers/functions.rs @@ -1,9 +1,6 @@ //! Backend-neutral execution history entry points and model validation. -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, pyfunction, - wrap_pyfunction, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::PyTuple; use sqlbuild_python_base::boundary::main::optional_text::optional_text; use sqlbuild_state::execution_history::main::project_runs::project_runs; @@ -13,29 +10,34 @@ use sqlbuild_state::execution_history::main::validate_page_limit::validate_page_ use sqlbuild_state::execution_history::main::validate_stored_event::validate_stored_event; use sqlbuild_state::execution_history::models::FilterSpec; -use crate::bindings::_helpers::execution_history::appends::{event_content, event_id}; -use crate::bindings::_helpers::execution_history::conversion::{ - checked, event_filter_spec, exact_int, projection_facts, run_filter_spec, run_records, - run_rows, utc_zone, -}; +use crate::execution_history::_helpers::appends::event_content; +use crate::execution_history::_helpers::appends::event_id; +use crate::execution_history::_helpers::conversion::checked; +use crate::execution_history::_helpers::conversion::event_filter_spec; +use crate::execution_history::_helpers::conversion::exact_int; +use crate::execution_history::_helpers::conversion::projection_facts; +use crate::execution_history::_helpers::conversion::run_filter_spec; +use crate::execution_history::_helpers::conversion::run_records; +use crate::execution_history::_helpers::conversion::run_rows; +use crate::execution_history::_helpers::conversion::utc_zone; use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Return the required stable ID from a known or opaque lifecycle event. #[pyfunction] -fn canonical_event_id(event: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn canonical_event_id(event: &Bound<'_, PyAny>) -> PyResult { event_id(event) } /// Return deterministic content for lifecycle event idempotency comparison. #[pyfunction] -fn canonical_event_content(event: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn canonical_event_content(event: &Bound<'_, PyAny>) -> PyResult { event_content(event) } /// Apply durable event facts in storage order to immutable run projections. #[pyfunction] #[pyo3(signature = (*, stored_events, current_runs = None))] -fn project_execution_runs<'py>( +pub(crate) fn project_execution_runs<'py>( py: Python<'py>, stored_events: &Bound<'py, PyAny>, current_runs: Option<&Bound<'py, PyAny>>, @@ -50,14 +52,14 @@ fn project_execution_runs<'py>( /// Validate a positive page limit no greater than the public maximum. #[pyfunction] -fn validate_history_page_limit(limit: &Bound<'_, PyAny>) -> Result<(), PyErr> { +pub(crate) fn validate_history_page_limit(limit: &Bound<'_, PyAny>) -> Result<(), PyErr> { checked(limit.py(), validate_page_limit(exact_int(limit))) } /// Append facts durably before publishing their rebuildable run projection. #[pyfunction] #[pyo3(signature = (*, event_log, run_storage, events))] -fn append_and_project<'py>( +pub(crate) fn append_and_project<'py>( event_log: &Bound<'py, PyAny>, run_storage: &Bound<'py, PyAny>, events: &Bound<'py, PyAny>, @@ -69,7 +71,7 @@ fn append_and_project<'py>( /// Validate a `StoredEvent`'s position, cursor and UTC receipt time. #[pyfunction] -fn validate_stored_history_event(record: &Bound<'_, PyAny>) -> Result<(), PyErr> { +pub(crate) fn validate_stored_history_event(record: &Bound<'_, PyAny>) -> Result<(), PyErr> { let py: Python<'_> = record.py(); let storage_order: i64 = native_field(record, "storage_order")?.extract()?; let cursor: String = optional_text(&native_field(record, "cursor")?)?.unwrap_or_default(); @@ -82,7 +84,7 @@ fn validate_stored_history_event(record: &Bound<'_, PyAny>) -> Result<(), PyErr> /// Validate an `EventFilter`. #[pyfunction] -fn validate_history_event_filter(event_filter: &Bound<'_, PyAny>) -> Result<(), PyErr> { +pub(crate) fn validate_history_event_filter(event_filter: &Bound<'_, PyAny>) -> Result<(), PyErr> { checked( event_filter.py(), validate_filter(FilterSpec::Event(&event_filter_spec(event_filter)?)), @@ -91,7 +93,7 @@ fn validate_history_event_filter(event_filter: &Bound<'_, PyAny>) -> Result<(), /// Validate a `RunFilter`. #[pyfunction] -fn validate_history_run_filter(run_filter: &Bound<'_, PyAny>) -> Result<(), PyErr> { +pub(crate) fn validate_history_run_filter(run_filter: &Bound<'_, PyAny>) -> Result<(), PyErr> { checked( run_filter.py(), validate_filter(FilterSpec::Run(&run_filter_spec(run_filter)?)), @@ -101,7 +103,10 @@ fn validate_history_run_filter(run_filter: &Bound<'_, PyAny>) -> Result<(), PyEr /// Validate an `EventPage` or `RunPage` next cursor against its records. #[pyfunction] #[pyo3(signature = (*, page, is_event_page))] -fn validate_history_page(page: &Bound<'_, PyAny>, is_event_page: bool) -> Result<(), PyErr> { +pub(crate) fn validate_history_page( + page: &Bound<'_, PyAny>, + is_event_page: bool, +) -> Result<(), PyErr> { let py: Python<'_> = page.py(); let records: Bound<'_, PyAny> = native_field(page, "records")?; let has_records: bool = records.is_truthy()?; @@ -122,15 +127,3 @@ fn validate_history_page(page: &Bound<'_, PyAny>, is_event_page: bool) -> Result validate_page(is_event_page, has_records, (next, last_text.as_deref())), ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(canonical_event_id, module)?)?; - module.add_function(wrap_pyfunction!(canonical_event_content, module)?)?; - module.add_function(wrap_pyfunction!(project_execution_runs, module)?)?; - module.add_function(wrap_pyfunction!(validate_history_page_limit, module)?)?; - module.add_function(wrap_pyfunction!(append_and_project, module)?)?; - module.add_function(wrap_pyfunction!(validate_stored_history_event, module)?)?; - module.add_function(wrap_pyfunction!(validate_history_event_filter, module)?)?; - module.add_function(wrap_pyfunction!(validate_history_run_filter, module)?)?; - module.add_function(wrap_pyfunction!(validate_history_page, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/getters.rs b/crates/sqlbuild-python-front/src/execution_history/_helpers/getters.rs similarity index 60% rename from crates/sqlbuild-python/src/bindings/_helpers/execution_history/getters.rs rename to crates/sqlbuild-python-front/src/execution_history/_helpers/getters.rs index 3a9ab803d7..57c608929b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/getters.rs +++ b/crates/sqlbuild-python-front/src/execution_history/_helpers/getters.rs @@ -3,24 +3,10 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::execution_history::postgres::PostgresExecutionHistory; -use crate::bindings::_helpers::execution_history::sqlite::SqliteExecutionHistory; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::execution_history::models::PostgresExecutionHistory; +use crate::execution_history::models::SqliteExecutionHistory; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[ - GetterEntry { - class: class_of::, - read: read_postgres_execution_history, - }, - GetterEntry { - class: class_of::, - read: read_sqlite_execution_history, - }, -]; - -fn read_postgres_execution_history<'py>( +pub(crate) fn read_postgres_execution_history<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -37,7 +23,7 @@ fn read_postgres_execution_history<'py>( }) } -fn read_sqlite_execution_history<'py>( +pub(crate) fn read_sqlite_execution_history<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-front/src/execution_history/_helpers/mod.rs b/crates/sqlbuild-python-front/src/execution_history/_helpers/mod.rs new file mode 100644 index 0000000000..8d5412aad7 --- /dev/null +++ b/crates/sqlbuild-python-front/src/execution_history/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod appends; +pub(crate) mod conversion; +pub(crate) mod functions; +pub(crate) mod getters; +pub(crate) mod postgres; +pub(crate) mod sqlite; +pub(crate) mod statements; diff --git a/crates/sqlbuild-python-front/src/execution_history/_helpers/postgres.rs b/crates/sqlbuild-python-front/src/execution_history/_helpers/postgres.rs new file mode 100644 index 0000000000..32a71f730f --- /dev/null +++ b/crates/sqlbuild-python-front/src/execution_history/_helpers/postgres.rs @@ -0,0 +1,104 @@ +//! `PostgresExecutionHistory`: deployed PostgreSQL event-log and run-projection storage. + +use crate::execution_history::main::utc::utc; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyDateTime; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_state::execution_history::constants::POSTGRES_RETRYABLE_SQLSTATES; +use sqlbuild_state::execution_history::models::RunRow; +use sqlbuild_state::execution_history::types::UtcTime; + +use crate::execution_history::_helpers::conversion::exact_int; +use crate::execution_history::_helpers::conversion::parsed_status; +use crate::execution_history::_helpers::conversion::storage_error; +use crate::execution_history::_helpers::conversion::utc_time; +use crate::execution_history::_helpers::statements::column; +use sqlbuild_python_base::boundary::main::imported::imported; + +pub(crate) const PSYCOPG_MODULE: &str = "psycopg"; +pub(crate) const PSYCOPG_ROWS_MODULE: &str = "psycopg.rows"; +pub(crate) const CLOSED: &str = "PostgreSQL execution history storage is closed"; +pub(crate) const MISSING_DRIVER: &str = "PostgreSQL execution history requires psycopg 3; install sqlbuild[postgres] or 'psycopg[binary]>=3.2'"; +pub(crate) const CONNECT_FAILED: &str = "cannot connect to PostgreSQL execution history; verify the secret-resolved DSN, network access, and sqlbuild[postgres] installation"; +pub(crate) const READ_FAILED: &str = "PostgreSQL execution history read failed"; +pub(crate) const METADATA_INVALID: &str = "PostgreSQL execution history schema metadata is invalid"; +pub(crate) const NAMESPACE_MISSING: &str = + "PostgreSQL execution history cursor namespace is missing"; +pub(crate) const STORAGE_ERROR: ExecutorHost = ExecutorHost::ExecutionHistoryStorageError; + +/// The `psycopg.Error` class. +fn driver_error_class(py: Python<'_>) -> PyResult> { + imported(py, PSYCOPG_MODULE, "Error") +} + +pub(crate) fn driver_failure(py: Python<'_>, error: &PyErr) -> PyResult { + Ok(error.is_instance(py, &driver_error_class(py)?)) +} + +/// Serialization failures and deadlocks retry while attempts remain. +pub(crate) fn retries_transaction(sqlstate: Option<&str>, attempts: i64, max_retries: i64) -> bool { + sqlstate.is_some_and(|state| POSTGRES_RETRYABLE_SQLSTATES.contains(&state)) + && attempts < max_retries +} + +/// `value` when it is an `int` that is not a `bool`. +pub(crate) fn storage_position(value: &Bound<'_, PyAny>, message: &str) -> PyResult { + exact_int(value).ok_or_else(|| storage_error(value.py(), message)) +} + +/// `value.astimezone(UTC)` when it is a `datetime`. +pub(crate) fn utc_datetime<'py>( + value: &Bound<'py, PyAny>, + message: &str, +) -> PyResult> { + let py: Python<'py> = value.py(); + if !value.is_instance_of::() { + return Err(storage_error(py, message)); + } + value.call_method1("astimezone", (utc(py)?,)) +} + +fn optional_utc(value: &Bound<'_, PyAny>) -> PyResult> { + if !value.is_instance_of::() { + return Ok(None); + } + utc_time(value).map(Some) +} + +fn text_column(row: &Bound<'_, PyAny>, name: &str) -> PyResult { + column(row, name)?.str()?.extract() +} + +/// One `sqlbuild_run_projection` row as a projected run. +pub(crate) fn stored_run(row: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = row.py(); + let created_at = utc_datetime( + &column(row, "created_at")?, + "PostgreSQL run timestamp is invalid", + )?; + let last_storage_order: i64 = storage_position( + &column(row, "last_event_storage_id")?, + "PostgreSQL run storage position is invalid", + )?; + let optional_str = |name: &str| -> PyResult> { + let value = column(row, name)?; + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.extract()?)) + }; + Ok(RunRow { + run_id: text_column(row, "run_id")?, + invocation_id: text_column(row, "invocation_id")?, + created_at: utc_time(&created_at)?, + status: parsed_status(py, &text_column(row, "status")?)?, + is_complete: column(row, "is_complete")?.is_truthy()?, + last_event_cursor: text_column(row, "last_event_cursor")?, + last_storage_order, + command: optional_str("command")?, + target: optional_str("target")?, + environment: optional_str("environment")?, + started_at: optional_utc(&column(row, "started_at")?)?, + ended_at: optional_utc(&column(row, "ended_at")?)?, + }) +} diff --git a/crates/sqlbuild-python-front/src/execution_history/_helpers/sqlite.rs b/crates/sqlbuild-python-front/src/execution_history/_helpers/sqlite.rs new file mode 100644 index 0000000000..ad05755c48 --- /dev/null +++ b/crates/sqlbuild-python-front/src/execution_history/_helpers/sqlite.rs @@ -0,0 +1,140 @@ +//! `SQLiteExecutionHistory`: project-local SQLite event-log and run-projection storage. + +use crate::execution_history::main::nullable_text::nullable_text; +use std::fs::DirBuilder; +use std::io::ErrorKind; +use std::path::Path; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_state::execution_history::constants::MEMORY_PATH; +use sqlbuild_state::execution_history::models::RunRow; +use sqlbuild_state::execution_history::types::UtcTime; + +use crate::execution_history::_helpers::conversion::parsed_status; +use crate::execution_history::_helpers::conversion::storage_error; +use crate::execution_history::_helpers::conversion::storage_error_from; +use crate::execution_history::_helpers::conversion::utc_time; +use crate::execution_history::_helpers::statements::column; +use sqlbuild_python_base::boundary::main::imported::imported; + +const SQLITE_MODULE: &str = "sqlite3"; +pub(crate) const PATHLIB_MODULE: &str = "pathlib"; +pub(crate) const CLOSED: &str = "SQLite execution history storage is closed"; +const NOT_AWARE: &str = "SQLite history timestamp is not timezone-aware"; +pub(crate) const INTEGRITY_CONFLICT: ExecutorHost = ExecutorHost::IntegrityConflictError; +pub(crate) const STORAGE_ERROR: ExecutorHost = ExecutorHost::ExecutionHistoryStorageError; + +/// `Path(path).parent.mkdir(mode=0o700, parents=True, exist_ok=True)` for a file-backed history. +pub(crate) fn prepare_history_directory(path: &str) -> std::io::Result<()> { + if path == MEMORY_PATH { + return Ok(()); + } + let Some(parent) = Path::new(path).parent() else { + return Ok(()); + }; + if parent.as_os_str().is_empty() || parent.is_dir() { + return Ok(()); + } + if let Some(grandparent) = parent.parent() + && !grandparent.as_os_str().is_empty() + { + std::fs::create_dir_all(grandparent)?; + } + match private_directory_builder().create(parent) { + Err(error) if error.kind() == ErrorKind::AlreadyExists && parent.is_dir() => Ok(()), + result => result, + } +} + +/// A directory builder with the owner-only mode Python's `os.mkdir(mode=0o700)` applies. +#[cfg(unix)] +fn private_directory_builder() -> DirBuilder { + use std::os::unix::fs::DirBuilderExt; + let mut builder: DirBuilder = DirBuilder::new(); + builder.mode(sqlbuild_state::execution_history::constants::SQLITE_DIRECTORY_MODE); + builder +} + +/// Windows ignores the directory mode, as `os.mkdir` does there. +#[cfg(not(unix))] +fn private_directory_builder() -> DirBuilder { + DirBuilder::new() +} + +/// `os.chmod(path, 0o600)` for a file-backed history. +pub(crate) fn restrict_history_file(path: &str) -> std::io::Result<()> { + if path == MEMORY_PATH { + return Ok(()); + } + set_owner_only(path) +} + +#[cfg(unix)] +fn set_owner_only(path: &str) -> std::io::Result<()> { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions( + path, + std::fs::Permissions::from_mode( + sqlbuild_state::execution_history::constants::SQLITE_FILE_MODE, + ), + ) +} + +/// `os.chmod(path, 0o600)` on Windows only keeps the file writable, which it already is. +#[cfg(not(unix))] +fn set_owner_only(_path: &str) -> std::io::Result<()> { + Ok(()) +} + +/// Whether `error` is a `sqlite3.Error`. +pub(crate) fn sqlite_failure(py: Python<'_>, error: &PyErr) -> PyResult { + Ok(error.is_instance(py, &imported(py, SQLITE_MODULE, "Error")?)) +} + +/// `result`, with a `sqlite3.Error` raised as a storage error `message` from it. +pub(crate) fn wrapped(py: Python<'_>, result: PyResult, message: &str) -> PyResult { + match result { + Err(error) if sqlite_failure(py, &error)? => Err(storage_error_from(py, message, error)), + result => result, + } +} + +/// A stored ISO timestamp as an aware UTC `datetime`. +pub(crate) fn parsed_timestamp<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let text: String = value.extract()?; + let parsed: Bound<'py, PyAny> = imported(py, "datetime", "datetime")? + .call_method1("fromisoformat", (text.replace('Z', "+00:00"),))?; + if parsed.call_method0("utcoffset")?.is_none() { + return Err(storage_error(py, NOT_AWARE)); + } + parsed.call_method1("astimezone", (imported(py, "datetime", "UTC")?,)) +} + +fn optional_timestamp(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + utc_time(&parsed_timestamp(value)?).map(Some) +} + +/// One `run_projection` row as a projected run. +pub(crate) fn stored_run(row: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = row.py(); + let status: String = column(row, "status")?.extract()?; + Ok(RunRow { + run_id: column(row, "run_id")?.extract()?, + invocation_id: column(row, "invocation_id")?.extract()?, + created_at: utc_time(&parsed_timestamp(&column(row, "created_at")?)?)?, + status: parsed_status(py, &status)?, + is_complete: column(row, "is_complete")?.is_truthy()?, + last_event_cursor: column(row, "last_event_cursor")?.extract()?, + last_storage_order: column(row, "last_storage_order")?.extract()?, + command: nullable_text(&column(row, "command")?)?, + target: nullable_text(&column(row, "target")?)?, + environment: nullable_text(&column(row, "environment")?)?, + started_at: optional_timestamp(&column(row, "started_at")?)?, + ended_at: optional_timestamp(&column(row, "ended_at")?)?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs b/crates/sqlbuild-python-front/src/execution_history/_helpers/statements.rs similarity index 77% rename from crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs rename to crates/sqlbuild-python-front/src/execution_history/_helpers/statements.rs index 56220bb4fd..a2d18f51a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs +++ b/crates/sqlbuild-python-front/src/execution_history/_helpers/statements.rs @@ -4,10 +4,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; use sqlbuild_state::execution_history::models::{Query, SqlParam}; -use crate::bindings::_helpers::execution_history::conversion::python_params; +use crate::execution_history::_helpers::conversion::python_params; use sqlbuild_python_base::warehouse_access::main::sqlite_params::sqlite_params; use sqlbuild_python_base::warehouse_access::models::SqliteConnection; -use sqlbuild_python_base::warehouse_access::models::SqliteCursor; use sqlbuild_python_base::warehouse_access::models::SqliteRow; /// `target.execute(sql, params)`. @@ -48,19 +47,6 @@ pub(crate) fn execute_query<'py>( execute(target, &query.sql, &query.params) } -/// The first row `cursor.fetchone()` returns, or `None`. -pub(crate) fn fetch_one<'py>(cursor: &Bound<'py, PyAny>) -> PyResult>> { - if let Ok(cursor) = cursor.cast::() { - return cursor - .get() - .next_row() - .map(|row| Bound::new(cursor.py(), row).map(Bound::into_any)) - .transpose(); - } - let row: Bound<'py, PyAny> = cursor.call_method0("fetchone")?; - Ok((!row.is_none()).then_some(row)) -} - /// Every row `cursor.fetchall()` returns. pub(crate) fn fetch_all<'py>(cursor: &Bound<'py, PyAny>) -> PyResult>> { cursor.call_method0("fetchall")?.try_iter()?.collect() diff --git a/crates/sqlbuild-python-front/src/execution_history/constants.rs b/crates/sqlbuild-python-front/src/execution_history/constants.rs new file mode 100644 index 0000000000..d10917c906 --- /dev/null +++ b/crates/sqlbuild-python-front/src/execution_history/constants.rs @@ -0,0 +1,20 @@ +//! Public constants of execution history. + +use crate::execution_history::_helpers::getters::read_postgres_execution_history; +use crate::execution_history::_helpers::getters::read_sqlite_execution_history; +use crate::execution_history::models::PostgresExecutionHistory; +use crate::execution_history::models::SqliteExecutionHistory; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_postgres_execution_history, + }, + GetterEntry { + class: class_of::, + read: read_sqlite_execution_history, + }, +]; diff --git a/crates/sqlbuild-python-front/src/execution_history/main/fetch_one.rs b/crates/sqlbuild-python-front/src/execution_history/main/fetch_one.rs new file mode 100644 index 0000000000..755f67b222 --- /dev/null +++ b/crates/sqlbuild-python-front/src/execution_history/main/fetch_one.rs @@ -0,0 +1,17 @@ +//! The first row `cursor.fetchone()` returns, or `None`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::warehouse_access::models::SqliteCursor; + +/// The first row `cursor.fetchone()` returns, or `None`. +pub fn fetch_one<'py>(cursor: &Bound<'py, PyAny>) -> PyResult>> { + if let Ok(cursor) = cursor.cast::() { + return cursor + .get() + .next_row() + .map(|row| Bound::new(cursor.py(), row).map(Bound::into_any)) + .transpose(); + } + let row: Bound<'py, PyAny> = cursor.call_method0("fetchone")?; + Ok((!row.is_none()).then_some(row)) +} diff --git a/crates/sqlbuild-python-front/src/execution_history/main/mod.rs b/crates/sqlbuild-python-front/src/execution_history/main/mod.rs new file mode 100644 index 0000000000..a03a257c75 --- /dev/null +++ b/crates/sqlbuild-python-front/src/execution_history/main/mod.rs @@ -0,0 +1,5 @@ +pub mod fetch_one; +pub mod nullable_text; +pub mod parsed_iso; +pub mod register_functions; +pub mod utc; diff --git a/crates/sqlbuild-python-front/src/execution_history/main/nullable_text.rs b/crates/sqlbuild-python-front/src/execution_history/main/nullable_text.rs new file mode 100644 index 0000000000..819ebb50cc --- /dev/null +++ b/crates/sqlbuild-python-front/src/execution_history/main/nullable_text.rs @@ -0,0 +1,11 @@ +//! `value` as text, or `None` when it is `None`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +/// `value` as text, or `None` when it is `None`. +pub fn nullable_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.extract()?)) +} diff --git a/crates/sqlbuild-python-front/src/execution_history/main/parsed_iso.rs b/crates/sqlbuild-python-front/src/execution_history/main/parsed_iso.rs new file mode 100644 index 0000000000..7f8e3366ca --- /dev/null +++ b/crates/sqlbuild-python-front/src/execution_history/main/parsed_iso.rs @@ -0,0 +1,18 @@ +//! `datetime.fromisoformat(text.replace("Z", "+00:00"))`, or `None` on `ValueError`. + +use crate::cursor::constants::DATETIME_MODULE; +use crate::execution_history::_helpers::conversion::UTC_OFFSET; +use crate::execution_history::_helpers::conversion::UTC_SUFFIX; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_python_base::boundary::main::imported::imported; + +/// `datetime.fromisoformat(text.replace("Z", "+00:00"))`, or `None` on `ValueError`. +pub fn parsed_iso<'py>(py: Python<'py>, text: &str) -> PyResult>> { + let normalized: String = text.replace(UTC_SUFFIX, UTC_OFFSET); + match imported(py, DATETIME_MODULE, "datetime")?.call_method1("fromisoformat", (normalized,)) { + Ok(value) => Ok(Some(value)), + Err(error) if error.is_instance_of::(py) => Ok(None), + Err(error) => Err(error), + } +} diff --git a/crates/sqlbuild-python-front/src/execution_history/main/register_functions.rs b/crates/sqlbuild-python-front/src/execution_history/main/register_functions.rs new file mode 100644 index 0000000000..94048ffe6f --- /dev/null +++ b/crates/sqlbuild-python-front/src/execution_history/main/register_functions.rs @@ -0,0 +1,25 @@ +//! Register the functions functions and classes. + +use crate::execution_history::_helpers::functions::append_and_project; +use crate::execution_history::_helpers::functions::canonical_event_content; +use crate::execution_history::_helpers::functions::canonical_event_id; +use crate::execution_history::_helpers::functions::project_execution_runs; +use crate::execution_history::_helpers::functions::validate_history_event_filter; +use crate::execution_history::_helpers::functions::validate_history_page; +use crate::execution_history::_helpers::functions::validate_history_page_limit; +use crate::execution_history::_helpers::functions::validate_history_run_filter; +use crate::execution_history::_helpers::functions::validate_stored_history_event; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, wrap_pyfunction}; + +/// Register the functions functions and classes. +pub fn register_functions(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(canonical_event_id, module)?)?; + module.add_function(wrap_pyfunction!(canonical_event_content, module)?)?; + module.add_function(wrap_pyfunction!(project_execution_runs, module)?)?; + module.add_function(wrap_pyfunction!(validate_history_page_limit, module)?)?; + module.add_function(wrap_pyfunction!(append_and_project, module)?)?; + module.add_function(wrap_pyfunction!(validate_stored_history_event, module)?)?; + module.add_function(wrap_pyfunction!(validate_history_event_filter, module)?)?; + module.add_function(wrap_pyfunction!(validate_history_run_filter, module)?)?; + module.add_function(wrap_pyfunction!(validate_history_page, module)?) +} diff --git a/crates/sqlbuild-python-front/src/execution_history/main/utc.rs b/crates/sqlbuild-python-front/src/execution_history/main/utc.rs new file mode 100644 index 0000000000..ffe7d175f4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/execution_history/main/utc.rs @@ -0,0 +1,10 @@ +//! `datetime.UTC`. + +use crate::cursor::constants::DATETIME_MODULE; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::boundary::main::imported::imported; + +/// `datetime.UTC`. +pub fn utc(py: Python<'_>) -> PyResult> { + imported(py, DATETIME_MODULE, "UTC") +} diff --git a/crates/sqlbuild-python-front/src/execution_history/mod.rs b/crates/sqlbuild-python-front/src/execution_history/mod.rs new file mode 100644 index 0000000000..9b7ecb955d --- /dev/null +++ b/crates/sqlbuild-python-front/src/execution_history/mod.rs @@ -0,0 +1,6 @@ +//! The execution history store on SQLite and PostgreSQL: run and event appends, statements, conversions and the Python functions over them. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-front/src/execution_history/models.rs b/crates/sqlbuild-python-front/src/execution_history/models.rs new file mode 100644 index 0000000000..9c0def6322 --- /dev/null +++ b/crates/sqlbuild-python-front/src/execution_history/models.rs @@ -0,0 +1,1833 @@ +//! Public records of execution history. + +use crate::execution_history::_helpers::appends::prepared_batch; +use crate::execution_history::_helpers::conversion::checked; +use crate::execution_history::_helpers::conversion::event_filter_spec; +use crate::execution_history::_helpers::conversion::exact_int; +use crate::execution_history::_helpers::conversion::history_error_of_kind; +use crate::execution_history::_helpers::conversion::new_namespace; +use crate::execution_history::_helpers::conversion::now; +use crate::execution_history::_helpers::conversion::projection_facts; +use crate::execution_history::_helpers::conversion::python_params; +use crate::execution_history::_helpers::conversion::rejection; +use crate::execution_history::_helpers::conversion::run_filter_spec; +use crate::execution_history::_helpers::conversion::run_records; +use crate::execution_history::_helpers::conversion::run_rows; +use crate::execution_history::_helpers::conversion::storage_error; +use crate::execution_history::_helpers::conversion::storage_error_from; +use crate::execution_history::_helpers::conversion::stored_event; +use crate::execution_history::_helpers::conversion::utc_time; +use crate::execution_history::_helpers::postgres::CONNECT_FAILED; +use crate::execution_history::_helpers::postgres::METADATA_INVALID; +use crate::execution_history::_helpers::postgres::MISSING_DRIVER; +use crate::execution_history::_helpers::postgres::NAMESPACE_MISSING; +use crate::execution_history::_helpers::postgres::PSYCOPG_MODULE; +use crate::execution_history::_helpers::postgres::PSYCOPG_ROWS_MODULE; +use crate::execution_history::_helpers::postgres::READ_FAILED; +use crate::execution_history::_helpers::postgres::driver_failure; +use crate::execution_history::_helpers::postgres::retries_transaction; +use crate::execution_history::_helpers::postgres::storage_position; +use crate::execution_history::_helpers::postgres::utc_datetime; +use crate::execution_history::_helpers::sqlite::INTEGRITY_CONFLICT; +use crate::execution_history::_helpers::sqlite::PATHLIB_MODULE; +use crate::execution_history::_helpers::sqlite::parsed_timestamp; +use crate::execution_history::_helpers::sqlite::prepare_history_directory; +use crate::execution_history::_helpers::sqlite::restrict_history_file; +use crate::execution_history::_helpers::sqlite::sqlite_failure; +use crate::execution_history::_helpers::sqlite::wrapped; +use crate::execution_history::_helpers::statements::column; +use crate::execution_history::_helpers::statements::execute; +use crate::execution_history::_helpers::statements::execute_plain; +use crate::execution_history::_helpers::statements::execute_query; +use crate::execution_history::_helpers::statements::fetch_all; +use crate::execution_history::_helpers::statements::tuple_of; +use crate::execution_history::main::fetch_one::fetch_one; +use crate::execution_history::main::nullable_text::nullable_text; +use crate::execution_history::main::parsed_iso::parsed_iso; +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyException; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::prelude::pyclass; +use pyo3::prelude::pymethods; +use pyo3::types::PyBool; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::boundary::main::keywords::keywords; +use sqlbuild_python_base::boundary::main::optional_text::optional_text; +use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_threads::models::ReentrantLock; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::warehouse_access::main::sqlite_params::sqlite_params; +use sqlbuild_python_base::warehouse_access::models::SqliteConnection; +use sqlbuild_python_base::warehouse_access::models::SqliteCursor; +use sqlbuild_state::execution_history::constants::DEFAULT_PAGE_LIMIT; +use sqlbuild_state::execution_history::constants::MEMORY_PATH; +use sqlbuild_state::execution_history::constants::POSTGRES_ADVISORY_LOCK; +use sqlbuild_state::execution_history::constants::POSTGRES_ALL_EVENTS; +use sqlbuild_state::execution_history::constants::POSTGRES_ALL_RUNS; +use sqlbuild_state::execution_history::constants::POSTGRES_CONNECT_TIMEOUT_SECONDS; +use sqlbuild_state::execution_history::constants::POSTGRES_DELETE_RUNS; +use sqlbuild_state::execution_history::constants::POSTGRES_EVENT_BY_ID; +use sqlbuild_state::execution_history::constants::POSTGRES_EVENT_EXISTS; +use sqlbuild_state::execution_history::constants::POSTGRES_HISTORY_LOCK_ID; +use sqlbuild_state::execution_history::constants::POSTGRES_MIGRATION_LOCK_ID; +use sqlbuild_state::execution_history::constants::POSTGRES_MIGRATIONS_EXIST; +use sqlbuild_state::execution_history::constants::POSTGRES_NAMESPACE; +use sqlbuild_state::execution_history::constants::POSTGRES_PING; +use sqlbuild_state::execution_history::constants::POSTGRES_RECORD_MIGRATION; +use sqlbuild_state::execution_history::constants::POSTGRES_RUN_BY_ID; +use sqlbuild_state::execution_history::constants::POSTGRES_RUN_EXISTS; +use sqlbuild_state::execution_history::constants::POSTGRES_SCHEMA; +use sqlbuild_state::execution_history::constants::POSTGRES_SCHEMA_VERSION; +use sqlbuild_state::execution_history::constants::POSTGRES_SESSION_TIMEOUT; +use sqlbuild_state::execution_history::constants::POSTGRES_STATEMENT_TIMEOUT_MS; +use sqlbuild_state::execution_history::constants::POSTGRES_TRANSACTION_RETRIES; +use sqlbuild_state::execution_history::constants::POSTGRES_TRANSACTION_TIMEOUT; +use sqlbuild_state::execution_history::constants::RUN_CURSOR_TIMESTAMP; +use sqlbuild_state::execution_history::constants::SCHEMA_VERSION; +use sqlbuild_state::execution_history::constants::SQLITE_ALL_EVENTS; +use sqlbuild_state::execution_history::constants::SQLITE_ALL_RUNS; +use sqlbuild_state::execution_history::constants::SQLITE_BEGIN; +use sqlbuild_state::execution_history::constants::SQLITE_BUSY_TIMEOUT_MS; +use sqlbuild_state::execution_history::constants::SQLITE_COMMIT; +use sqlbuild_state::execution_history::constants::SQLITE_DELETE_RUNS; +use sqlbuild_state::execution_history::constants::SQLITE_EVENT_BY_ID; +use sqlbuild_state::execution_history::constants::SQLITE_EVENT_BY_STORAGE_ID; +use sqlbuild_state::execution_history::constants::SQLITE_EVENT_EXISTS; +use sqlbuild_state::execution_history::constants::SQLITE_FOREIGN_KEYS; +use sqlbuild_state::execution_history::constants::SQLITE_HEALTHY_CHECK_RESULT; +use sqlbuild_state::execution_history::constants::SQLITE_INSERT_NAMESPACE; +use sqlbuild_state::execution_history::constants::SQLITE_INSERT_SCHEMA_VERSION; +use sqlbuild_state::execution_history::constants::SQLITE_JOURNAL_MODE; +use sqlbuild_state::execution_history::constants::SQLITE_METADATA_EXISTS; +use sqlbuild_state::execution_history::constants::SQLITE_METADATA_TABLE; +use sqlbuild_state::execution_history::constants::SQLITE_METADATA_VALUE; +use sqlbuild_state::execution_history::constants::SQLITE_NAMESPACE_KEY; +use sqlbuild_state::execution_history::constants::SQLITE_QUICK_CHECK; +use sqlbuild_state::execution_history::constants::SQLITE_ROLLBACK; +use sqlbuild_state::execution_history::constants::SQLITE_RUN_BY_ID; +use sqlbuild_state::execution_history::constants::SQLITE_RUN_EXISTS; +use sqlbuild_state::execution_history::constants::SQLITE_RUN_INSERT; +use sqlbuild_state::execution_history::constants::SQLITE_SCHEMA; +use sqlbuild_state::execution_history::constants::SQLITE_SCHEMA_VERSION_KEY; +use sqlbuild_state::execution_history::constants::UNKNOWN_EVENT_CURSOR; +use sqlbuild_state::execution_history::constants::UNKNOWN_RUN_CURSOR; +use sqlbuild_state::execution_history::errors::HistoryErrorKind; +use sqlbuild_state::execution_history::main::decoded_run_cursor::decoded_run_cursor; +use sqlbuild_state::execution_history::main::event_cursor::event_cursor; +use sqlbuild_state::execution_history::main::event_cursor_position::event_cursor_position; +use sqlbuild_state::execution_history::main::event_insert_query::event_insert_query; +use sqlbuild_state::execution_history::main::event_page_query::event_page_query; +use sqlbuild_state::execution_history::main::project_runs::project_runs; +use sqlbuild_state::execution_history::main::run_cursor::run_cursor; +use sqlbuild_state::execution_history::main::run_insert_query::run_insert_query; +use sqlbuild_state::execution_history::main::run_page_query::run_page_query; +use sqlbuild_state::execution_history::main::schema_migration_needed::schema_migration_needed; +use sqlbuild_state::execution_history::main::sqlite_database_path::sqlite_database_path; +use sqlbuild_state::execution_history::main::validate_page_limit::validate_page_limit; +use sqlbuild_state::execution_history::main::validate_postgres_settings::validate_postgres_settings; +use sqlbuild_state::execution_history::main::validate_schema_target::validate_schema_target; +use sqlbuild_state::execution_history::main::validate_stored_content::validate_stored_content; +use sqlbuild_state::execution_history::models::Dialect; +use sqlbuild_state::execution_history::models::PostgresSettings; +use sqlbuild_state::execution_history::models::PreparedEvent; +use sqlbuild_state::execution_history::models::Query; +use sqlbuild_state::execution_history::models::RunCursorKey; +use sqlbuild_state::execution_history::models::RunRow; +use sqlbuild_state::execution_history::models::SqlParam; +use sqlbuild_state::execution_history::models::SqliteLocation; +use std::sync::OnceLock; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicI64; +use std::sync::atomic::Ordering; + +/// Deployed PostgreSQL implementation of event-log and run-projection contracts. +#[pyclass( + module = "sqlbuild._native", + name = "PostgresExecutionHistory", + frozen, + subclass, + dict +)] +#[derive(Debug)] +pub struct PostgresExecutionHistory { + pub(crate) statement_timeout_ms: OnceLock, + pub(crate) max_transaction_retries: OnceLock, + pub(crate) closed: AtomicBool, + pub(crate) project_calls: AtomicI64, +} + +impl PostgresExecutionHistory { + pub(crate) fn ensure_open(&self, py: Python<'_>) -> PyResult<()> { + if self.closed.load(Ordering::SeqCst) { + return Err(storage_error( + py, + crate::execution_history::_helpers::postgres::CLOSED, + )); + } + Ok(()) + } + + /// Run `action` in a transaction with the statement timeout, retrying serialization failures. + pub(crate) fn transactional<'py, T>( + &self, + slf: &Bound<'py, Self>, + action: impl Fn(&Bound<'py, PyAny>) -> PyResult, + failure_message: &str, + ) -> PyResult { + let py: Python<'py> = slf.py(); + let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; + let timeout: String = self + .statement_timeout_ms + .get() + .copied() + .unwrap_or(POSTGRES_STATEMENT_TIMEOUT_MS) + .to_string(); + let max_retries: i64 = self + .max_transaction_retries + .get() + .copied() + .unwrap_or(POSTGRES_TRANSACTION_RETRIES); + let mut attempts: i64 = 0; + loop { + let result: PyResult = within(&connection.call_method0("transaction")?, |_| { + within(&connection.call_method0("cursor")?, |cursor| { + execute( + cursor, + POSTGRES_TRANSACTION_TIMEOUT, + &[SqlParam::Text(timeout.clone())], + )?; + action(cursor) + }) + }); + let error: PyErr = match result { + Ok(value) => return Ok(value), + Err(error) => error, + }; + if history_error_of_kind( + py, + &error, + crate::execution_history::_helpers::postgres::STORAGE_ERROR, + )? || !driver_failure(py, &error)? + { + return Err(error); + } + let sqlstate: Option = + optional_text(&native_field(error.value(py), "sqlstate")?)?; + if retries_transaction(sqlstate.as_deref(), attempts, max_retries) { + attempts += 1; + continue; + } + return Err(storage_error(py, failure_message)); + } + } + + /// `query` through a fresh cursor, with driver failures reported as read failures. + pub(crate) fn read_rows<'py>( + slf: &Bound<'py, Self>, + query: &Query, + ) -> PyResult>> { + let py: Python<'py> = slf.py(); + let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; + let rows = within(&connection.call_method0("cursor")?, |cursor| { + execute_query(cursor, query)?; + fetch_all(cursor) + }); + match rows { + Err(error) if driver_failure(py, &error)? => Err(storage_error(py, READ_FAILED)), + rows => rows, + } + } + + pub(crate) fn with_cursor<'py, T>( + slf: &Bound<'py, Self>, + cursor: Option<&Bound<'py, PyAny>>, + body: impl FnOnce(&Bound<'py, PyAny>) -> PyResult, + ) -> PyResult { + if let Some(cursor) = cursor { + return body(cursor); + } + let owned: Bound<'py, PyAny> = native_field(slf, "_connection")?.call_method0("cursor")?; + let result: PyResult = body(&owned); + let closed = owned.call_method0("close"); + let value: T = result?; + closed?; + Ok(value) + } + + pub(crate) fn schema_version_in<'py>( + slf: &Bound<'py, Self>, + cursor: Option<&Bound<'py, PyAny>>, + ) -> PyResult { + let py: Python<'py> = slf.py(); + Self::with_cursor(slf, cursor, |active| { + let read = (|| -> PyResult { + execute_plain(active, POSTGRES_MIGRATIONS_EXIST)?; + let Some(exists) = fetch_one(active)? else { + return Ok(0); + }; + if exists.get_item("table_name")?.is_none() { + return Ok(0); + } + execute_plain(active, POSTGRES_SCHEMA_VERSION)?; + let row = fetch_one(active)?; + let value = row.map(|row| row.get_item("schema_version")).transpose()?; + value + .as_ref() + .and_then(exact_int) + .ok_or_else(|| storage_error(py, METADATA_INVALID)) + })(); + match read { + Err(error) + if history_error_of_kind( + py, + &error, + crate::execution_history::_helpers::postgres::STORAGE_ERROR, + )? => + { + Err(error) + } + Err(error) if error.is_instance_of::(py) => { + Err(storage_error(py, METADATA_INVALID)) + } + read => read, + } + }) + } + + pub(crate) fn namespace_in<'py>( + slf: &Bound<'py, Self>, + cursor: Option<&Bound<'py, PyAny>>, + ) -> PyResult { + let py: Python<'py> = slf.py(); + Self::with_cursor(slf, cursor, |active| { + execute_plain(active, POSTGRES_NAMESPACE)?; + let row = fetch_one(active)?; + let namespace = match row { + Some(row) => optional_text(&row.get_item("storage_namespace")?)?, + None => None, + }; + namespace.ok_or_else(|| storage_error(py, NAMESPACE_MISSING)) + }) + } + + pub(crate) fn stored_event<'py>( + row: &Bound<'py, PyAny>, + namespace: &str, + ) -> PyResult> { + let py: Python<'py> = row.py(); + let storage_id: i64 = storage_position( + &column(row, "storage_id")?, + "PostgreSQL event storage position is invalid", + )?; + let received_at = utc_datetime( + &column(row, "received_at")?, + "PostgreSQL event timestamp is invalid", + )?; + let payload: Bound<'py, PyAny> = column(row, "payload_json")?; + if optional_text(&payload)?.is_none() { + return Err(storage_error( + py, + "PostgreSQL canonical event content is invalid", + )); + } + stored_event( + py, + ( + storage_id, + event_cursor(Dialect::Postgres, namespace, storage_id), + ), + received_at, + &payload, + ) + } + + pub(crate) fn stored_events<'py>( + slf: &Bound<'py, Self>, + rows: &[Bound<'py, PyAny>], + ) -> PyResult>> { + if rows.is_empty() { + return Ok(Vec::new()); + } + let namespace: String = Self::namespace_in(slf, None)?; + rows.iter() + .map(|row| Self::stored_event(row, &namespace)) + .collect() + } + + pub(crate) fn read_all_runs(cursor: &Bound<'_, PyAny>) -> PyResult> { + execute_plain(cursor, POSTGRES_ALL_RUNS)?; + fetch_all(cursor)? + .iter() + .map(crate::execution_history::_helpers::postgres::stored_run) + .collect() + } + + pub(crate) fn lock_history(cursor: &Bound<'_, PyAny>) -> PyResult<()> { + execute( + cursor, + POSTGRES_ADVISORY_LOCK, + &[SqlParam::Integer(POSTGRES_HISTORY_LOCK_ID)], + )?; + Ok(()) + } + + pub(crate) fn append_via<'py>( + slf: &Bound<'py, Self>, + cursor: &Bound<'py, PyAny>, + events: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + native_field(slf, "_append_events")?.call( + (), + Some(&keywords( + py, + &[("cursor", cursor.clone()), ("events", events.clone())], + )?), + ) + } + + pub(crate) fn replace_via<'py>( + slf: &Bound<'py, Self>, + cursor: &Bound<'py, PyAny>, + projected: &Bound<'py, PyAny>, + ) -> PyResult<()> { + let py: Python<'py> = slf.py(); + native_field(slf, "_replace_projection")?.call( + (), + Some(&keywords( + py, + &[("cursor", cursor.clone()), ("projected", projected.clone())], + )?), + )?; + Ok(()) + } + + pub(crate) fn migrate(&self, slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + self.transactional( + slf, + |cursor| { + execute( + cursor, + POSTGRES_ADVISORY_LOCK, + &[SqlParam::Integer(POSTGRES_MIGRATION_LOCK_ID)], + )?; + let current: i64 = Self::schema_version_in(slf, Some(cursor))?; + if !checked(py, schema_migration_needed(Dialect::Postgres, current))? { + return Ok(()); + } + for statement in POSTGRES_SCHEMA { + execute_plain(cursor, statement)?; + } + execute( + cursor, + POSTGRES_RECORD_MIGRATION, + &[ + SqlParam::Integer(SCHEMA_VERSION), + SqlParam::Text(new_namespace(py)?), + ], + )?; + Ok(()) + }, + "PostgreSQL execution history schema migration failed", + ) + } + + pub(crate) fn run_cursor_key( + slf: &Bound<'_, Self>, + cursor: Option<&Bound<'_, PyAny>>, + ) -> PyResult> { + let py: Python<'_> = slf.py(); + let Some(cursor) = cursor.filter(|cursor| !cursor.is_none()) else { + return Ok(None); + }; + let text: String = optional_text(cursor)?.unwrap_or_default(); + let namespace: String = Self::namespace_in(slf, None)?; + let decoded = checked(py, decoded_run_cursor(Dialect::Postgres, &text, &namespace))?; + let invalid = || rejection(py, HistoryErrorKind::InvalidCursor, RUN_CURSOR_TIMESTAMP); + let parsed = imported(py, "datetime", "datetime")? + .call_method1("fromisoformat", (decoded.created_at_text.as_str(),)); + let parsed = match parsed { + Ok(parsed) => parsed, + Err(error) if error.is_instance_of::(py) => { + return Err(invalid()); + } + Err(error) => return Err(error), + }; + if parsed.call_method0("utcoffset")?.is_none() { + return Err(invalid()); + } + let created_at = utc_time(&parsed)?; + let rows = Self::read_rows( + slf, + &Query { + sql: POSTGRES_RUN_EXISTS.to_owned(), + params: vec![ + SqlParam::Timestamp(created_at), + SqlParam::Text(decoded.run_id.clone()), + ], + }, + )?; + if rows.is_empty() { + return Err(rejection( + py, + HistoryErrorKind::InvalidCursor, + UNKNOWN_RUN_CURSOR, + )); + } + Ok(Some(RunCursorKey { + created_at, + created_at_text: decoded.created_at_text, + run_id: decoded.run_id, + })) + } +} + +#[pymethods] +impl PostgresExecutionHistory { + #[new] + #[pyo3(signature = (*_args, **_kwargs))] + pub(crate) fn new(_args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>) -> Self { + Self { + statement_timeout_ms: OnceLock::new(), + max_transaction_retries: OnceLock::new(), + closed: AtomicBool::new(true), + project_calls: AtomicI64::new(0), + } + } + + #[pyo3(signature = (dsn, **options))] + pub(crate) fn __init__( + slf: &Bound<'_, Self>, + dsn: &Bound<'_, PyAny>, + options: Option>, + ) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let this: &Self = slf.get(); + let options = KeywordArguments::read( + py, + "__init__", + options, + ( + &[], + &[ + "connect_timeout_seconds", + "statement_timeout_ms", + "max_transaction_retries", + ], + ), + )?; + let setting = |name: &str, default: i64| -> PyResult> { + if options.passed(name)? { + return Ok(exact_int(&options.value(name)?)); + } + Ok(Some(default)) + }; + let settings = PostgresSettings { + dsn: optional_text(dsn)?, + connect_timeout_seconds: setting( + "connect_timeout_seconds", + POSTGRES_CONNECT_TIMEOUT_SECONDS, + )?, + statement_timeout_ms: setting("statement_timeout_ms", POSTGRES_STATEMENT_TIMEOUT_MS)?, + max_transaction_retries: setting( + "max_transaction_retries", + POSTGRES_TRANSACTION_RETRIES, + )?, + }; + checked(py, validate_postgres_settings(&settings))?; + let (psycopg, dict_row) = match ( + py.import(PSYCOPG_MODULE), + imported(py, PSYCOPG_ROWS_MODULE, "dict_row"), + ) { + (Ok(psycopg), Ok(dict_row)) => (psycopg, dict_row), + (Err(error), _) | (_, Err(error)) + if error.is_instance_of::(py) => + { + return Err(storage_error(py, MISSING_DRIVER)); + } + (Err(error), _) | (_, Err(error)) => return Err(error), + }; + let statement_timeout: i64 = settings + .statement_timeout_ms + .unwrap_or(POSTGRES_STATEMENT_TIMEOUT_MS); + let _ = this.max_transaction_retries.set( + settings + .max_transaction_retries + .unwrap_or(POSTGRES_TRANSACTION_RETRIES), + ); + this.closed.store(false, Ordering::SeqCst); + let mut connection: Option> = None; + let opened: PyResult<()> = (|| { + let opened: Bound<'_, PyAny> = native_field(&psycopg, "connect")?.call( + (dsn,), + Some(&keywords( + py, + &[ + ( + "connect_timeout", + settings.connect_timeout_seconds.into_bound_py_any(py)?, + ), + ("autocommit", PyBool::new(py, true).to_owned().into_any()), + ("row_factory", dict_row.clone()), + ], + )?), + )?; + connection = Some(opened.clone()); + slf.setattr("_connection", &opened)?; + within(&opened.call_method0("cursor")?, |cursor| { + execute( + cursor, + POSTGRES_SESSION_TIMEOUT, + &[SqlParam::Text(statement_timeout.to_string())], + ) + })?; + let _ = this.statement_timeout_ms.set(statement_timeout); + this.migrate(slf)?; + slf.call_method0("reconcile")?; + Ok(()) + })(); + let Err(error) = opened else { + return Ok(()); + }; + if !error.is_instance_of::(py) { + return Err(error); + } + if let Some(connection) = connection { + let _ = connection.call_method0("close"); + } + this.closed.store(true, Ordering::SeqCst); + if history_error_of_kind( + py, + &error, + crate::execution_history::_helpers::postgres::STORAGE_ERROR, + )? { + return Err(error); + } + Err(storage_error(py, CONNECT_FAILED)) + } + + pub(crate) fn __repr__(slf: &Bound<'_, Self>) -> PyResult { + let state: &str = if slf.get().closed.load(Ordering::SeqCst) { + "closed" + } else { + "open" + }; + let name: String = slf.get_type().getattr("__name__")?.extract()?; + Ok(format!("{name}(state='{state}')")) + } + + /// The number of incremental projection calls. + #[getter] + pub fn project_calls(&self) -> i64 { + self.project_calls.load(Ordering::SeqCst) + } + + pub(crate) fn append_event<'py>( + slf: &Bound<'py, Self>, + event: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + slf.get().ensure_open(py)?; + slf.call_method1("append_events", (tuple_of(py, vec![event.clone()])?,))? + .get_item(0) + } + + pub(crate) fn append_events<'py>( + slf: &Bound<'py, Self>, + events: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + let this: &Self = slf.get(); + this.ensure_open(py)?; + let pending: Bound<'py, PyTuple> = + PyTuple::new(py, events.try_iter()?.collect::>>()?)?; + this.transactional( + slf, + |cursor| { + Self::lock_history(cursor)?; + Self::append_via(slf, cursor, pending.as_any()) + }, + "PostgreSQL event append failed", + ) + } + + #[pyo3(signature = (*, event_filter, after_cursor = None, limit = None))] + #[pyo3(name = "get_events")] + pub(crate) fn stored_events_page<'py>( + slf: &Bound<'py, Self>, + event_filter: &Bound<'py, PyAny>, + after_cursor: Option<&Bound<'py, PyAny>>, + limit: Option<&Bound<'py, PyAny>>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + slf.get().ensure_open(py)?; + let limit: Option = limit.map_or(Some(DEFAULT_PAGE_LIMIT), exact_int); + checked(py, validate_page_limit(limit))?; + let limit: i64 = limit.unwrap_or(DEFAULT_PAGE_LIMIT); + let after: i64 = match after_cursor.filter(|cursor| !cursor.is_none()) { + None => 0, + Some(cursor) => { + let text: String = optional_text(cursor)?.unwrap_or_default(); + let namespace: String = Self::namespace_in(slf, None)?; + let position: i64 = checked( + py, + event_cursor_position(Dialect::Postgres, Some(&text), &namespace), + )?; + let rows = Self::read_rows( + slf, + &Query { + sql: POSTGRES_EVENT_EXISTS.to_owned(), + params: vec![SqlParam::Integer(position)], + }, + )?; + if rows.is_empty() { + return Err(rejection( + py, + HistoryErrorKind::InvalidCursor, + UNKNOWN_EVENT_CURSOR, + )); + } + position + } + }; + let spec = event_filter_spec(event_filter)?; + let rows = Self::read_rows( + slf, + &event_page_query(Dialect::Postgres, after, &spec, limit), + )?; + let has_more: bool = rows.len() as i64 > limit; + let page: Vec> = rows.into_iter().take(limit as usize).collect(); + let records: Vec> = Self::stored_events(slf, &page)?; + let next_cursor: Bound<'py, PyAny> = match records.last() { + Some(record) => native_field(record, "cursor")?, + None => py.None().into_bound(py), + }; + call_with( + &host_entry(py, ExecutorHost::EventPage)?, + &[ + ("records", tuple_of(py, records)?.into_any()), + ("next_cursor", next_cursor), + ("has_more", has_more.into_bound_py_any(py)?), + ], + ) + } + + #[pyo3(name = "get_run")] + pub(crate) fn run_record<'py>( + slf: &Bound<'py, Self>, + run_id: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + slf.get().ensure_open(py)?; + let run_id: Option = nullable_text(run_id)?; + let params: Vec = vec![run_id.map_or(SqlParam::Null, SqlParam::Text)]; + let rows = Self::read_rows( + slf, + &Query { + sql: POSTGRES_RUN_BY_ID.to_owned(), + params, + }, + )?; + match rows.first() { + Some(row) => crate::execution_history::_helpers::conversion::run_record( + py, + &crate::execution_history::_helpers::postgres::stored_run(row)?, + ), + None => Ok(py.None().into_bound(py)), + } + } + + #[pyo3(signature = (*, run_filter, after_cursor = None, limit = None))] + #[pyo3(name = "get_runs")] + pub(crate) fn run_page<'py>( + slf: &Bound<'py, Self>, + run_filter: &Bound<'py, PyAny>, + after_cursor: Option<&Bound<'py, PyAny>>, + limit: Option<&Bound<'py, PyAny>>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + slf.get().ensure_open(py)?; + let limit: Option = limit.map_or(Some(DEFAULT_PAGE_LIMIT), exact_int); + checked(py, validate_page_limit(limit))?; + let limit: i64 = limit.unwrap_or(DEFAULT_PAGE_LIMIT); + let after = Self::run_cursor_key(slf, after_cursor)?; + let spec = run_filter_spec(run_filter)?; + let rows = Self::read_rows( + slf, + &run_page_query(Dialect::Postgres, after.as_ref(), &spec, limit), + )?; + let runs: Vec = rows + .iter() + .take(limit as usize) + .map(crate::execution_history::_helpers::postgres::stored_run) + .collect::>()?; + let next_cursor: Bound<'py, PyAny> = match runs.last() { + Some(run) => { + let namespace: String = Self::namespace_in(slf, None)?; + checked(py, run_cursor(Dialect::Postgres, &namespace, run))? + .into_bound_py_any(py)? + } + None => py.None().into_bound(py), + }; + call_with( + &host_entry(py, ExecutorHost::RunPage)?, + &[ + ("records", run_records(py, &runs)?.into_any()), + ("next_cursor", next_cursor), + ( + "has_more", + (rows.len() as i64 > limit).into_bound_py_any(py)?, + ), + ], + ) + } + + pub(crate) fn project<'py>( + slf: &Bound<'py, Self>, + stored_events: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + let this: &Self = slf.get(); + this.ensure_open(py)?; + let pending: Bound<'py, PyTuple> = + PyTuple::new(py, stored_events.try_iter()?.collect::>>()?)?; + this.project_calls.fetch_add(1, Ordering::SeqCst); + this.transactional( + slf, + |cursor| { + Self::lock_history(cursor)?; + let facts = projection_facts(pending.as_any())?; + let projected = checked(py, project_runs(facts, Self::read_all_runs(cursor)?))?; + let records: Bound<'py, PyTuple> = run_records(py, &projected)?; + Self::replace_via(slf, cursor, records.as_any())?; + Ok(records) + }, + "PostgreSQL run projection publication failed", + ) + } + + pub(crate) fn rebuild_from_events<'py>( + slf: &Bound<'py, Self>, + stored_events: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + let this: &Self = slf.get(); + this.ensure_open(py)?; + let pending: Bound<'py, PyTuple> = + PyTuple::new(py, stored_events.try_iter()?.collect::>>()?)?; + this.transactional( + slf, + |cursor| { + Self::lock_history(cursor)?; + let projected = checked( + py, + project_runs(projection_facts(pending.as_any())?, Vec::new()), + )?; + let records: Bound<'py, PyTuple> = run_records(py, &projected)?; + Self::replace_via(slf, cursor, records.as_any())?; + Ok(records) + }, + "PostgreSQL run projection rebuild failed", + ) + } + + /// Atomically append immutable facts and publish their run projection. + pub(crate) fn append_and_project<'py>( + slf: &Bound<'py, Self>, + events: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + let this: &Self = slf.get(); + this.ensure_open(py)?; + let pending: Bound<'py, PyTuple> = + PyTuple::new(py, events.try_iter()?.collect::>>()?)?; + this.transactional( + slf, + |cursor| { + Self::lock_history(cursor)?; + let stored: Bound<'py, PyAny> = Self::append_via(slf, cursor, pending.as_any())?; + let projected = checked( + py, + project_runs(projection_facts(&stored)?, Self::read_all_runs(cursor)?), + )?; + let records: Bound<'py, PyTuple> = run_records(py, &projected)?; + Self::replace_via(slf, cursor, records.as_any())?; + Ok(stored) + }, + "PostgreSQL atomic append and projection failed", + ) + } + + /// Rebuild the disposable run projection from all durable event facts. + pub(crate) fn reconcile<'py>(slf: &Bound<'py, Self>) -> PyResult> { + let py: Python<'py> = slf.py(); + let this: &Self = slf.get(); + this.ensure_open(py)?; + this.transactional( + slf, + |cursor| { + Self::lock_history(cursor)?; + let stored: Bound<'py, PyAny> = native_field(slf, "_read_all_events")? + .call((), Some(&keywords(py, &[("cursor", cursor.clone())])?))?; + let projected = checked(py, project_runs(projection_facts(&stored)?, Vec::new()))?; + let records: Bound<'py, PyTuple> = run_records(py, &projected)?; + Self::replace_via(slf, cursor, records.as_any())?; + Ok(records) + }, + "PostgreSQL reconciliation failed", + ) + } + + /// Verify connectivity and the supported schema revision. + pub(crate) fn check_health(slf: &Bound<'_, Self>) -> PyResult { + let py: Python<'_> = slf.py(); + slf.get().ensure_open(py)?; + let healthy = (|| -> PyResult { + let connection: Bound<'_, PyAny> = native_field(slf, "_connection")?; + let row = within(&connection.call_method0("cursor")?, |cursor| { + execute_plain(cursor, POSTGRES_PING)?; + fetch_one(cursor) + })?; + let version: i64 = slf.call_method0("get_schema_version")?.extract()?; + Ok(row.is_some() && version == SCHEMA_VERSION) + })(); + match healthy { + Err(error) if driver_failure(py, &error)? => Ok(false), + healthy => healthy, + } + } + + #[pyo3(name = "get_schema_version")] + pub(crate) fn schema_version(slf: &Bound<'_, Self>) -> PyResult { + slf.get().ensure_open(slf.py())?; + Self::schema_version_in(slf, None) + } + + #[pyo3(signature = (*, target_version = None))] + pub(crate) fn upgrade_schema( + slf: &Bound<'_, Self>, + target_version: Option, + ) -> PyResult { + let py: Python<'_> = slf.py(); + let this: &Self = slf.get(); + this.ensure_open(py)?; + let target: i64 = target_version.unwrap_or(SCHEMA_VERSION); + checked(py, validate_schema_target(Dialect::Postgres, target))?; + this.migrate(slf)?; + Ok(target) + } + + pub(crate) fn close(slf: &Bound<'_, Self>) -> PyResult<()> { + let this: &Self = slf.get(); + if this.closed.load(Ordering::SeqCst) { + return Ok(()); + } + native_field(slf, "_connection")?.call_method0("close")?; + this.closed.store(true, Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn dispose(slf: &Bound<'_, Self>) -> PyResult<()> { + slf.call_method0("close")?; + Ok(()) + } + + pub(crate) fn __enter__<'py>(slf: &Bound<'py, Self>) -> PyResult> { + slf.get().ensure_open(slf.py())?; + Ok(slf.clone()) + } + + #[pyo3(signature = (*_exc_info))] + pub(crate) fn __exit__( + slf: &Bound<'_, Self>, + _exc_info: &Bound<'_, PyTuple>, + ) -> PyResult> { + slf.call_method0("close")?; + Ok(None) + } + + #[pyo3(signature = (*, cursor, events))] + pub(crate) fn _append_events<'py>( + slf: &Bound<'py, Self>, + cursor: &Bound<'py, PyAny>, + events: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + let prepared: Vec = prepared_batch(events)?; + let mut stored: Vec> = Vec::with_capacity(prepared.len()); + let mut namespace: Option = None; + let received_at = now(py)?; + for event in &prepared { + execute_query( + cursor, + &event_insert_query(Dialect::Postgres, event, &received_at), + )?; + let mut row = fetch_one(cursor)?; + if row.is_none() { + execute( + cursor, + POSTGRES_EVENT_BY_ID, + &[SqlParam::Text(event.event_id.clone())], + )?; + row = fetch_one(cursor)?; + } + let Some(row) = row else { + return Err(storage_error( + py, + "PostgreSQL event append did not produce a durable fact", + )); + }; + let digest: Option = optional_text(&column(&row, "content_digest")?)?; + let payload: Option = optional_text(&column(&row, "payload_json")?)?; + checked( + py, + validate_stored_content(event, (digest.as_deref(), payload.as_deref())), + )?; + if namespace.is_none() { + namespace = Some(Self::namespace_in(slf, Some(cursor))?); + } + stored.push(Self::stored_event( + &row, + namespace.as_deref().unwrap_or_default(), + )?); + } + tuple_of(py, stored) + } + + #[pyo3(signature = (*, cursor, projected))] + pub(crate) fn _replace_projection( + _slf: &Bound<'_, Self>, + cursor: &Bound<'_, PyAny>, + projected: &Bound<'_, PyAny>, + ) -> PyResult<()> { + execute_plain(cursor, POSTGRES_DELETE_RUNS)?; + for run in run_rows(projected)? { + execute_query(cursor, &run_insert_query(Dialect::Postgres, &run))?; + } + Ok(()) + } + + #[pyo3(signature = (*, cursor))] + pub(crate) fn _read_all_events<'py>( + slf: &Bound<'py, Self>, + cursor: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + execute_plain(cursor, POSTGRES_ALL_EVENTS)?; + let rows: Vec> = fetch_all(cursor)?; + let stored: Vec> = if rows.is_empty() { + Vec::new() + } else { + let namespace: String = Self::namespace_in(slf, Some(cursor))?; + rows.iter() + .map(|row| Self::stored_event(row, &namespace)) + .collect::>()? + }; + tuple_of(py, stored) + } +} + +/// Project-local SQLite implementation of event-log and run-projection storage contracts. +#[pyclass( + module = "sqlbuild._native", + name = "SQLiteExecutionHistory", + frozen, + subclass, + dict +)] +#[derive(Debug)] +pub struct SqliteExecutionHistory { + pub(crate) path: OnceLock, + pub(crate) lock: ReentrantLock, + pub(crate) closed: AtomicBool, + pub(crate) project_calls: AtomicI64, +} + +impl SqliteExecutionHistory { + pub(crate) fn database_path(&self) -> &str { + self.path.get().map_or(MEMORY_PATH, String::as_str) + } + + pub(crate) fn with_lock<'py, T>( + &self, + py: Python<'py>, + body: impl FnOnce() -> PyResult, + ) -> PyResult { + self.lock.with_lock(py, body) + } + + pub(crate) fn ensure_open(&self, py: Python<'_>) -> PyResult<()> { + if self.closed.load(Ordering::SeqCst) { + return Err(storage_error( + py, + crate::execution_history::_helpers::sqlite::CLOSED, + )); + } + Ok(()) + } + + /// Run `body` in `BEGIN IMMEDIATE`, committing on success and rolling back on failure. + pub(crate) fn transaction<'py, T>( + &self, + slf: &Bound<'py, Self>, + body: impl FnOnce(&Bound<'py, PyAny>) -> PyResult, + ) -> PyResult { + let py: Python<'py> = slf.py(); + self.with_lock(py, || { + self.ensure_open(py)?; + let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; + execute_plain(&connection, SQLITE_BEGIN)?; + let value: T = match body(&connection) { + Ok(value) => value, + Err(error) => { + let _ = execute_plain(&connection, SQLITE_ROLLBACK); + return Err(error); + } + }; + if let Err(error) = execute_plain(&connection, SQLITE_COMMIT) { + if sqlite_failure(py, &error)? + && match connection.cast::() { + Ok(connection) => connection.get().in_transaction(py)?, + Err(_) => native_field(&connection, "in_transaction")?.is_truthy()?, + } + { + let _ = execute_plain(&connection, SQLITE_ROLLBACK); + } + return Err(error); + } + Ok(value) + }) + } + + pub(crate) fn metadata_value<'py>( + connection: &Bound<'py, PyAny>, + key: &str, + ) -> PyResult>> { + let row = fetch_one(&execute( + connection, + SQLITE_METADATA_VALUE, + &[SqlParam::Text(key.to_owned())], + )?)?; + row.map(|row| row.get_item(0)).transpose() + } + + pub(crate) fn metadata_version(connection: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = connection.py(); + let exists = fetch_one(&execute( + connection, + SQLITE_METADATA_EXISTS, + &[SqlParam::Text(SQLITE_METADATA_TABLE.to_owned())], + )?)?; + if exists.is_none() { + return Ok(0); + } + let Some(value) = Self::metadata_value(connection, SQLITE_SCHEMA_VERSION_KEY)? else { + return Ok(0); + }; + let invalid = || storage_error(py, "SQLite history schema metadata is invalid"); + match optional_text(&value)? { + Some(text) => text.trim().parse::().map_err(|_| invalid()), + None => exact_int(&value).ok_or_else(invalid), + } + } + + pub(crate) fn namespace(connection: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = connection.py(); + match Self::metadata_value(connection, SQLITE_NAMESPACE_KEY)? { + Some(value) => Ok(value.str()?.extract()?), + None => Err(storage_error( + py, + "SQLite history cursor namespace is missing", + )), + } + } + + pub(crate) fn stored_event<'py>( + row: &Bound<'py, PyAny>, + namespace: &str, + ) -> PyResult> { + let py: Python<'py> = row.py(); + let storage_id: i64 = column(row, "storage_id")?.extract()?; + stored_event( + py, + ( + storage_id, + event_cursor(Dialect::Sqlite, namespace, storage_id), + ), + parsed_timestamp(&column(row, "received_at")?)?, + &column(row, "payload_json")?, + ) + } + + pub(crate) fn read_all_runs(connection: &Bound<'_, PyAny>) -> PyResult> { + execute_plain(connection, SQLITE_ALL_RUNS)? + .try_iter()? + .map(|row| { + row.and_then(|row| crate::execution_history::_helpers::sqlite::stored_run(&row)) + }) + .collect() + } + + pub(crate) fn replace_projection( + connection: &Bound<'_, PyAny>, + projected: &[RunRow], + ) -> PyResult<()> { + let py: Python<'_> = connection.py(); + execute_plain(connection, SQLITE_DELETE_RUNS)?; + if let Ok(connection) = connection.cast::() { + for run in projected { + connection.get().execute_native( + py, + SQLITE_RUN_INSERT, + &sqlite_params(&run_insert_query(Dialect::Sqlite, run).params)?, + )?; + } + return Ok(()); + } + let rows: Vec> = projected + .iter() + .map(|run| { + python_params(py, &run_insert_query(Dialect::Sqlite, run).params) + .map(Bound::into_any) + }) + .collect::>()?; + connection.call_method1("executemany", (SQLITE_RUN_INSERT, tuple_of(py, rows)?))?; + Ok(()) + } + + pub(crate) fn append_prepared<'py>( + connection: &Bound<'py, PyAny>, + prepared: &[PreparedEvent], + ) -> PyResult>> { + let py: Python<'py> = connection.py(); + let mut stored: Vec> = Vec::with_capacity(prepared.len()); + let mut namespace: Option = None; + for event in prepared { + let existing = fetch_one(&execute( + connection, + SQLITE_EVENT_BY_ID, + &[SqlParam::Text(event.event_id.clone())], + )?)?; + let row: Bound<'py, PyAny> = match existing { + Some(existing) => { + let digest: Option = + optional_text(&column(&existing, "content_digest")?)?; + let payload: Option = + optional_text(&column(&existing, "payload_json")?)?; + checked( + py, + validate_stored_content(event, (digest.as_deref(), payload.as_deref())), + )?; + existing + } + None => { + let query = event_insert_query(Dialect::Sqlite, event, &now(py)?); + let cursor: Bound<'py, PyAny> = execute_query(connection, &query)?; + let storage_id: i64 = match cursor.cast::() { + Ok(cursor) => cursor.get().last_row_id, + Err(_) => native_field(&cursor, "lastrowid")?.extract()?, + }; + fetch_one(&execute( + connection, + SQLITE_EVENT_BY_STORAGE_ID, + &[SqlParam::Integer(storage_id)], + )?)? + .unwrap_or_else(|| py.None().into_bound(py)) + } + }; + if namespace.is_none() { + namespace = Some(Self::namespace(connection)?); + } + let namespace: &str = namespace.as_deref().unwrap_or_default(); + stored.push(Self::stored_event(&row, namespace)?); + } + Ok(stored) + } + + pub(crate) fn append_events_in<'py>( + connection: &Bound<'py, PyAny>, + events: &Bound<'py, PyAny>, + ) -> PyResult>> { + let prepared: Vec = prepared_batch(events)?; + Self::append_prepared(connection, &prepared) + } + + pub(crate) fn migrate_in(&self, slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let connection: Bound<'_, PyAny> = native_field(slf, "_connection")?; + let current: i64 = Self::metadata_version(&connection)?; + if !checked(py, schema_migration_needed(Dialect::Sqlite, current))? { + return Ok(()); + } + let namespace: String = new_namespace(py)?; + let migrated = self.transaction(slf, |connection| { + for statement in SQLITE_SCHEMA { + execute_plain(connection, statement)?; + } + execute( + connection, + SQLITE_INSERT_NAMESPACE, + &[ + SqlParam::Text(SQLITE_NAMESPACE_KEY.to_owned()), + SqlParam::Text(namespace.clone()), + ], + )?; + execute( + connection, + SQLITE_INSERT_SCHEMA_VERSION, + &[ + SqlParam::Text(SQLITE_SCHEMA_VERSION_KEY.to_owned()), + SqlParam::Text(SCHEMA_VERSION.to_string()), + ], + )?; + Ok(()) + }); + wrapped(py, migrated, "SQLite history schema migration failed") + } + + pub(crate) fn run_cursor_key( + connection: &Bound<'_, PyAny>, + cursor: &Bound<'_, PyAny>, + ) -> PyResult> { + let py: Python<'_> = connection.py(); + if cursor.is_none() { + return Ok(None); + } + let text: String = optional_text(cursor)?.unwrap_or_default(); + let namespace: String = Self::namespace(connection)?; + let decoded = checked(py, decoded_run_cursor(Dialect::Sqlite, &text, &namespace))?; + let invalid_timestamp = + || rejection(py, HistoryErrorKind::InvalidCursor, RUN_CURSOR_TIMESTAMP); + let created_at = match parsed_iso(py, &decoded.created_at_text)? { + Some(parsed) if !parsed.call_method0("utcoffset")?.is_none() => utc_time(&parsed)?, + _ => return Err(invalid_timestamp()), + }; + let exists = fetch_one(&execute( + connection, + SQLITE_RUN_EXISTS, + &[ + SqlParam::Text(decoded.created_at_text.clone()), + SqlParam::Text(decoded.run_id.clone()), + ], + )?)?; + if exists.is_none() { + return Err(rejection( + py, + HistoryErrorKind::InvalidCursor, + UNKNOWN_RUN_CURSOR, + )); + } + Ok(Some(RunCursorKey { + created_at, + created_at_text: decoded.created_at_text, + run_id: decoded.run_id, + })) + } +} + +#[pymethods] +impl SqliteExecutionHistory { + #[new] + #[pyo3(signature = (*_args, **_kwargs))] + pub(crate) fn new(_args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>) -> Self { + Self { + path: OnceLock::new(), + lock: ReentrantLock::default(), + closed: AtomicBool::new(true), + project_calls: AtomicI64::new(0), + } + } + + #[pyo3(signature = (project_dir = None, *, path = None, busy_timeout_ms = None))] + pub(crate) fn __init__( + slf: &Bound<'_, Self>, + project_dir: Option<&Bound<'_, PyAny>>, + path: Option<&Bound<'_, PyAny>>, + busy_timeout_ms: Option<&Bound<'_, PyAny>>, + ) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let this: &Self = slf.get(); + let fspath = |value: Option<&Bound<'_, PyAny>>| -> PyResult> { + value + .filter(|value| !value.is_none()) + .map(|value| imported(py, "os", "fspath")?.call1((value,))?.extract()) + .transpose() + }; + let project_dir: Option = fspath(project_dir)?; + let path: Option = fspath(path)?; + let working_directory: String = if project_dir.is_none() && path.is_none() { + std::env::current_dir()?.to_string_lossy().into_owned() + } else { + String::new() + }; + let location = SqliteLocation { + project_dir, + path, + working_directory, + busy_timeout_ms: busy_timeout_ms.map_or(Some(SQLITE_BUSY_TIMEOUT_MS), exact_int), + }; + let database: String = checked(py, sqlite_database_path(&location))?; + let busy_timeout_ms: i64 = location.busy_timeout_ms.unwrap_or(SQLITE_BUSY_TIMEOUT_MS); + let _ = this.path.set(database.clone()); + this.closed.store(false, Ordering::SeqCst); + let mut connection: Option> = None; + let opened: PyResult<()> = (|| { + prepare_history_directory(&database)?; + let opened = Bound::new(py, SqliteConnection::open(py, &database)?)?.into_any(); + connection = Some(opened.clone()); + slf.setattr("_connection", &opened)?; + this.with_lock(py, || { + execute_plain(&opened, &format!("PRAGMA busy_timeout = {busy_timeout_ms}"))?; + execute_plain(&opened, SQLITE_FOREIGN_KEYS)?; + if database != MEMORY_PATH { + execute_plain(&opened, SQLITE_JOURNAL_MODE)?.call_method0("fetchone")?; + } + slf.call_method0("_migrate")?; + Ok(()) + })?; + restrict_history_file(&database)?; + slf.call_method0("reconcile")?; + Ok(()) + })(); + let Err(error) = opened else { + return Ok(()); + }; + let is_storage: bool = history_error_of_kind( + py, + &error, + crate::execution_history::_helpers::sqlite::STORAGE_ERROR, + )?; + let is_open_failure: bool = + error.is_instance_of::(py) || sqlite_failure(py, &error)?; + if !is_storage && !is_open_failure { + return Err(error); + } + if let Some(connection) = connection { + let _ = connection.call_method0("close"); + } + this.closed.store(true, Ordering::SeqCst); + if is_storage { + return Err(error); + } + Err(storage_error_from( + py, + &format!("cannot open SQLite history at {database}"), + error, + )) + } + + /// The database path, or `None` for an in-memory database. + #[getter] + pub fn path<'py>(&self, py: Python<'py>) -> PyResult> { + let path: &str = self.database_path(); + if path == MEMORY_PATH { + return Ok(py.None().into_bound(py)); + } + imported(py, PATHLIB_MODULE, "Path")?.call1((path,)) + } + + /// The number of incremental projection calls. + #[getter] + pub fn project_calls(&self) -> i64 { + self.project_calls.load(Ordering::SeqCst) + } + + pub(crate) fn append_event<'py>( + slf: &Bound<'py, Self>, + event: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + let this: &Self = slf.get(); + this.with_lock(py, || { + this.ensure_open(py)?; + slf.call_method1("append_events", (tuple_of(py, vec![event.clone()])?,))? + .get_item(0) + }) + } + + pub(crate) fn append_events<'py>( + slf: &Bound<'py, Self>, + events: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + let this: &Self = slf.get(); + this.with_lock(py, || { + this.ensure_open(py)?; + let pending: Bound<'py, PyTuple> = + PyTuple::new(py, events.try_iter()?.collect::>>()?)?; + let appended = this.transaction(slf, |connection| { + Self::append_events_in(connection, pending.as_any()) + }); + let stored = Self::without_conflicts(py, appended, "SQLite event append failed")?; + tuple_of(py, stored) + }) + } + + #[pyo3(signature = (*, event_filter, after_cursor = None, limit = None))] + #[pyo3(name = "get_events")] + pub(crate) fn stored_events_page<'py>( + slf: &Bound<'py, Self>, + event_filter: &Bound<'py, PyAny>, + after_cursor: Option<&Bound<'py, PyAny>>, + limit: Option<&Bound<'py, PyAny>>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + let this: &Self = slf.get(); + this.with_lock(py, || { + this.ensure_open(py)?; + let limit: Option = limit.map_or(Some(DEFAULT_PAGE_LIMIT), exact_int); + checked(py, validate_page_limit(limit))?; + let limit: i64 = limit.unwrap_or(DEFAULT_PAGE_LIMIT); + let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; + let after: i64 = Self::event_position(&connection, after_cursor)?; + let spec = event_filter_spec(event_filter)?; + let query = event_page_query(Dialect::Sqlite, after, &spec, limit); + let rows: Vec> = wrapped( + py, + execute_query(&connection, &query) + .and_then(|cursor| cursor.try_iter()?.collect::>>()), + "SQLite event read failed", + )?; + let has_more: bool = rows.len() as i64 > limit; + let mut records: Vec> = Vec::new(); + if !rows.is_empty() { + let namespace: String = Self::namespace(&connection)?; + for row in rows.iter().take(limit as usize) { + records.push(Self::stored_event(row, &namespace)?); + } + } + let next_cursor: Bound<'py, PyAny> = match records.last() { + Some(record) => native_field(record, "cursor")?, + None => py.None().into_bound(py), + }; + sqlbuild_python_base::boundary::main::call_with::call_with( + &host_entry(py, ExecutorHost::EventPage)?, + &[ + ("records", tuple_of(py, records)?.into_any()), + ("next_cursor", next_cursor), + ("has_more", has_more.into_bound_py_any(py)?), + ], + ) + }) + } + + #[pyo3(name = "get_run")] + pub(crate) fn run_record<'py>( + slf: &Bound<'py, Self>, + run_id: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + let this: &Self = slf.get(); + this.with_lock(py, || { + this.ensure_open(py)?; + let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; + let row = + fetch_one(&connection.call_method1("execute", (SQLITE_RUN_BY_ID, (run_id,)))?)?; + match row { + Some(row) => crate::execution_history::_helpers::conversion::run_record( + py, + &crate::execution_history::_helpers::sqlite::stored_run(&row)?, + ), + None => Ok(py.None().into_bound(py)), + } + }) + } + + #[pyo3(signature = (*, run_filter, after_cursor = None, limit = None))] + #[pyo3(name = "get_runs")] + pub(crate) fn run_page<'py>( + slf: &Bound<'py, Self>, + run_filter: &Bound<'py, PyAny>, + after_cursor: Option<&Bound<'py, PyAny>>, + limit: Option<&Bound<'py, PyAny>>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + let this: &Self = slf.get(); + this.with_lock(py, || { + this.ensure_open(py)?; + let limit: Option = limit.map_or(Some(DEFAULT_PAGE_LIMIT), exact_int); + checked(py, validate_page_limit(limit))?; + let limit: i64 = limit.unwrap_or(DEFAULT_PAGE_LIMIT); + let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; + let none = py.None().into_bound(py); + let after = Self::run_cursor_key(&connection, after_cursor.unwrap_or(&none))?; + let spec = run_filter_spec(run_filter)?; + let query = run_page_query(Dialect::Sqlite, after.as_ref(), &spec, limit); + let rows: Vec> = execute_query(&connection, &query)? + .try_iter()? + .collect::>()?; + let runs: Vec = rows + .iter() + .take(limit as usize) + .map(crate::execution_history::_helpers::sqlite::stored_run) + .collect::>()?; + let next_cursor: Bound<'py, PyAny> = match runs.last() { + Some(run) => { + let namespace: String = Self::namespace(&connection)?; + checked(py, run_cursor(Dialect::Sqlite, &namespace, run))? + .into_bound_py_any(py)? + } + None => py.None().into_bound(py), + }; + sqlbuild_python_base::boundary::main::call_with::call_with( + &host_entry(py, ExecutorHost::RunPage)?, + &[ + ("records", run_records(py, &runs)?.into_any()), + ("next_cursor", next_cursor), + ( + "has_more", + (rows.len() as i64 > limit).into_bound_py_any(py)?, + ), + ], + ) + }) + } + + pub(crate) fn project<'py>( + slf: &Bound<'py, Self>, + stored_events: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + let this: &Self = slf.get(); + this.with_lock(py, || { + this.ensure_open(py)?; + this.project_calls.fetch_add(1, Ordering::SeqCst); + let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; + let current: Vec = Self::read_all_runs(&connection)?; + let facts = projection_facts(stored_events)?; + let projected: Vec = checked(py, project_runs(facts, current))?; + let records: Bound<'py, PyTuple> = run_records(py, &projected)?; + slf.call_method1("_publish_projection", (&records,))?; + Ok(records) + }) + } + + pub(crate) fn rebuild_from_events<'py>( + slf: &Bound<'py, Self>, + stored_events: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + let this: &Self = slf.get(); + this.with_lock(py, || { + this.ensure_open(py)?; + let facts = projection_facts(stored_events)?; + let projected: Vec = checked(py, project_runs(facts, Vec::new()))?; + let records: Bound<'py, PyTuple> = run_records(py, &projected)?; + slf.call_method1("_publish_projection", (&records,))?; + Ok(records) + }) + } + + /// Atomically append immutable facts and publish their run projection. + pub(crate) fn append_and_project<'py>( + slf: &Bound<'py, Self>, + events: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = slf.py(); + let this: &Self = slf.get(); + this.with_lock(py, || { + this.ensure_open(py)?; + let pending: Bound<'py, PyTuple> = + PyTuple::new(py, events.try_iter()?.collect::>>()?)?; + let appended = this.transaction(slf, |connection| { + let stored: Vec> = + Self::append_events_in(connection, pending.as_any())?; + let stored_tuple: Bound<'py, PyTuple> = tuple_of(py, stored.clone())?; + let facts = projection_facts(stored_tuple.as_any())?; + let projected = checked(py, project_runs(facts, Self::read_all_runs(connection)?))?; + Self::replace_projection(connection, &projected)?; + Ok(stored) + }); + let stored = Self::without_conflicts( + py, + appended, + "SQLite atomic append and projection failed", + )?; + tuple_of(py, stored) + }) + } + + /// Rebuild the disposable run projection from all durable event facts. + pub(crate) fn reconcile<'py>(slf: &Bound<'py, Self>) -> PyResult> { + let py: Python<'py> = slf.py(); + let this: &Self = slf.get(); + this.with_lock(py, || { + this.ensure_open(py)?; + let reconciled = this.transaction(slf, |connection| { + let records: Bound<'py, PyAny> = slf.call_method0("_read_all_events")?; + let facts = projection_facts(&records)?; + let projected: Vec = checked(py, project_runs(facts, Vec::new()))?; + Self::replace_projection(connection, &projected)?; + run_records(py, &projected) + }); + wrapped(py, reconciled, "SQLite reconciliation failed") + }) + } + + /// Verify connectivity, integrity, and the supported schema revision. + pub(crate) fn check_health(slf: &Bound<'_, Self>) -> PyResult { + let py: Python<'_> = slf.py(); + let this: &Self = slf.get(); + this.with_lock(py, || { + this.ensure_open(py)?; + let connection: Bound<'_, PyAny> = native_field(slf, "_connection")?; + let Some(result) = fetch_one(&execute_plain(&connection, SQLITE_QUICK_CHECK)?)? else { + return Ok(false); + }; + if optional_text(&result.get_item(0)?)?.as_deref() != Some(SQLITE_HEALTHY_CHECK_RESULT) + { + return Ok(false); + } + let version: i64 = slf.call_method0("get_schema_version")?.extract()?; + Ok(version == SCHEMA_VERSION) + }) + } + + #[pyo3(name = "get_schema_version")] + pub(crate) fn schema_version(slf: &Bound<'_, Self>) -> PyResult { + let py: Python<'_> = slf.py(); + let this: &Self = slf.get(); + this.with_lock(py, || { + this.ensure_open(py)?; + Self::metadata_version(&native_field(slf, "_connection")?) + }) + } + + #[pyo3(signature = (*, target_version = None))] + pub(crate) fn upgrade_schema( + slf: &Bound<'_, Self>, + target_version: Option, + ) -> PyResult { + let py: Python<'_> = slf.py(); + let this: &Self = slf.get(); + this.with_lock(py, || { + this.ensure_open(py)?; + let target: i64 = target_version.unwrap_or(SCHEMA_VERSION); + checked(py, validate_schema_target(Dialect::Sqlite, target))?; + slf.call_method0("_migrate")?; + Ok(target) + }) + } + + pub(crate) fn close(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let this: &Self = slf.get(); + this.with_lock(py, || { + if this.closed.load(Ordering::SeqCst) { + return Ok(()); + } + let closed = native_field(slf, "_connection") + .and_then(|connection| connection.call_method0("close")); + this.closed.store(true, Ordering::SeqCst); + closed.map(|_| ()) + }) + } + + pub(crate) fn dispose(slf: &Bound<'_, Self>) -> PyResult<()> { + slf.call_method0("close")?; + Ok(()) + } + + pub(crate) fn __enter__<'py>(slf: &Bound<'py, Self>) -> PyResult> { + let py: Python<'py> = slf.py(); + let this: &Self = slf.get(); + this.with_lock(py, || this.ensure_open(py))?; + Ok(slf.clone()) + } + + #[pyo3(signature = (*_exc_info))] + pub(crate) fn __exit__( + slf: &Bound<'_, Self>, + _exc_info: &Bound<'_, PyTuple>, + ) -> PyResult> { + slf.call_method0("close")?; + Ok(None) + } + + pub(crate) fn _migrate(slf: &Bound<'_, Self>) -> PyResult<()> { + slf.get().migrate_in(slf) + } + + pub(crate) fn _read_all_events<'py>(slf: &Bound<'py, Self>) -> PyResult> { + let py: Python<'py> = slf.py(); + let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; + let rows: Vec> = execute_plain(&connection, SQLITE_ALL_EVENTS)? + .try_iter()? + .collect::>()?; + let mut records: Vec> = Vec::with_capacity(rows.len()); + if !rows.is_empty() { + let namespace: String = Self::namespace(&connection)?; + for row in &rows { + records.push(Self::stored_event(row, &namespace)?); + } + } + tuple_of(py, records) + } + + pub(crate) fn _publish_projection( + slf: &Bound<'_, Self>, + projected: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let runs: Vec = run_rows(projected)?; + let published = slf.get().transaction(slf, |connection| { + Self::replace_projection(connection, &runs) + }); + wrapped(py, published, "SQLite run projection publication failed") + } +} + +impl SqliteExecutionHistory { + /// `result` with integrity conflicts kept and `sqlite3.Error` wrapped as `message`. + pub(crate) fn without_conflicts( + py: Python<'_>, + result: PyResult, + message: &str, + ) -> PyResult { + match result { + Err(error) if history_error_of_kind(py, &error, INTEGRITY_CONFLICT)? => Err(error), + result => wrapped(py, result, message), + } + } + + pub(crate) fn event_position( + connection: &Bound<'_, PyAny>, + cursor: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + let py: Python<'_> = connection.py(); + let Some(cursor) = cursor.filter(|cursor| !cursor.is_none()) else { + return Ok(0); + }; + let text: String = optional_text(cursor)?.unwrap_or_default(); + let namespace: String = Self::namespace(connection)?; + let position: i64 = checked( + py, + event_cursor_position(Dialect::Sqlite, Some(&text), &namespace), + )?; + let exists = fetch_one(&execute( + connection, + SQLITE_EVENT_EXISTS, + &[SqlParam::Integer(position)], + )?)?; + if exists.is_none() { + return Err(rejection( + py, + HistoryErrorKind::InvalidCursor, + UNKNOWN_EVENT_CURSOR, + )); + } + Ok(position) + } +} diff --git a/crates/sqlbuild-python-front/src/lib.rs b/crates/sqlbuild-python-front/src/lib.rs index 183d051c5e..6393f84a48 100644 --- a/crates/sqlbuild-python-front/src/lib.rs +++ b/crates/sqlbuild-python-front/src/lib.rs @@ -11,11 +11,13 @@ pub mod contracts; pub mod cost_context; pub mod cursor; pub mod discovery; +pub mod execution_history; pub mod macro_bridge; pub mod model_config; pub mod native_host; pub mod refactor; pub mod render; +pub mod resource_costs; pub mod scopes; pub mod sql_test_glue; pub mod sqltext; diff --git a/crates/sqlbuild-python-front/src/resource_costs/_helpers/mod.rs b/crates/sqlbuild-python-front/src/resource_costs/_helpers/mod.rs new file mode 100644 index 0000000000..7d7aae7125 --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod resource_costs_state; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs b/crates/sqlbuild-python-front/src/resource_costs/_helpers/resource_costs_state.rs similarity index 53% rename from crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs rename to crates/sqlbuild-python-front/src/resource_costs/_helpers/resource_costs_state.rs index 1b6af182fe..347a1823b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs +++ b/crates/sqlbuild-python-front/src/resource_costs/_helpers/resource_costs_state.rs @@ -1,10 +1,11 @@ //! DDL, appends, latest reads and pruning of the display-only resource costs table. +use crate::resource_costs::models::CostRunIdentity; use chrono::DateTime; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyFloat, PyInt, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyInt; use sqlbuild_core::constants::RESOURCE_COSTS_TABLE_NAME; use sqlbuild_core::cost_command::models::CostCoverage; use sqlbuild_core::cost_report::models::CostSummary; @@ -12,35 +13,31 @@ use sqlbuild_state::events::main::resource_cost_observation_id::resource_cost_ob use sqlbuild_state::resource_costs::constants::{ NULLABLE_RESOURCE_COST_COLUMNS, RESOURCE_COST_COLUMN_TYPES, TOTAL_RESOURCE_TYPE, }; -use sqlbuild_state::resource_costs::main::latest_run_costs_sql::latest_run_costs_sql; use sqlbuild_state::resource_costs::main::prune_resource_costs_sql::prune_resource_costs_sql; use sqlbuild_state::resource_costs::models::{ResourceCostKey, RunCostAttempt}; -use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; -use crate::bindings::_helpers::execution_history::conversion::{parsed_iso, utc}; +use crate::execution_history::main::parsed_iso::parsed_iso; +use crate::execution_history::main::utc::utc; +use crate::native_host::main::fixed_state_table_sql::fixed_state_table_sql; +use crate::native_host::main::integer_literal::integer_literal; +use crate::native_host::main::literal::literal; +use crate::native_host::main::optional_text::optional_text; +use crate::native_host::main::state_table_name::state_table_name; +use crate::native_host::models::Location; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::native_host::constants::ROWS_PER_STATEMENT; -use sqlbuild_python_front::native_host::main::execute::execute; -use sqlbuild_python_front::native_host::main::fixed_state_table_sql::fixed_state_table_sql; -use sqlbuild_python_front::native_host::main::integer_literal::integer_literal; -use sqlbuild_python_front::native_host::main::literal::literal; -use sqlbuild_python_front::native_host::main::optional_text::optional_text; -use sqlbuild_python_front::native_host::main::state_table_name::state_table_name; -use sqlbuild_python_front::native_host::models::Location; /// Rate source of a price the project configured, the only one shown as USD. const CONFIGURED_RATE_SOURCE: &str = "configured"; /// The adapter method that marks the resource costs table as supported. -const CREATE_TABLE_METHOD: &str = "render_create_resource_costs_table_sql"; +pub(crate) const CREATE_TABLE_METHOD: &str = "render_create_resource_costs_table_sql"; /// Collection budget of each re-read of an older run, which a build waits for. -const REFRESH_BUDGET_SECONDS: f64 = 20.0; +pub(crate) const REFRESH_BUDGET_SECONDS: f64 = 20.0; /// Create-if-missing DDL for the resource costs table, with adapter-rendered column types. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name, render_framework_type))] -fn build_resource_costs_create_table_sql( +pub(crate) fn build_resource_costs_create_table_sql( database: &Bound<'_, PyAny>, schema: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, @@ -60,7 +57,7 @@ fn build_resource_costs_create_table_sql( /// SQL deleting observations beyond the newest `retain_versions` per run, resource and warehouse. #[pyfunction] #[pyo3(signature = (*, database, schema, render_qualified_name, retain_versions))] -fn build_resource_costs_prune_sql( +pub(crate) fn build_resource_costs_prune_sql( database: &Bound<'_, PyAny>, schema: &Bound<'_, PyAny>, render_qualified_name: &Bound<'_, PyAny>, @@ -75,16 +72,7 @@ fn build_resource_costs_prune_sql( .unwrap_or_default()) } -/// The run a collection belongs to, as the run row records it. -pub(crate) struct CostRunIdentity<'a> { - pub(crate) run_id: &'a str, - pub(crate) orchestration_run_id: Option<&'a str>, - pub(crate) target_name: Option<&'a str>, - pub(crate) started_at: &'a str, - pub(crate) completed_at: &'a str, -} - -fn table(adapter: &Bound<'_, PyAny>, location: &Location) -> PyResult> { +pub(crate) fn table(adapter: &Bound<'_, PyAny>, location: &Location) -> PyResult> { let keywords = location.keywords(adapter.py()); state_table_name( &native_field(adapter, "render_qualified_name")?, @@ -108,7 +96,7 @@ struct ResourceRow<'a> { } /// The run row and one row per resource and warehouse of one collection attempt. -fn observation_cells( +pub(crate) fn observation_cells( py: Python<'_>, (run, attempt): (&CostRunIdentity<'_>, i64), summary: &CostSummary, @@ -214,7 +202,7 @@ fn utc_text(cell: &Bound<'_, PyAny>) -> PyResult { aware.call_method0("isoformat")?.extract() } -fn run_attempt(row: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn run_attempt(row: &Bound<'_, PyAny>) -> PyResult { let cell = |index: usize| row.get_item(index); let text = |index: usize| -> PyResult { Ok(optional_text(&cell(index)?)?.unwrap_or_default()) @@ -242,161 +230,3 @@ fn run_attempt(row: &Bound<'_, PyAny>) -> PyResult { started_epoch, }) } - -/// A target's resource costs table, open on one connection for one build's publication. -pub(crate) struct ResourceCostStore<'py> { - finalization: Bound<'py, PyAny>, - adapter: Bound<'py, PyAny>, - connection: Bound<'py, PyAny>, - location: Location, - table: String, -} - -impl<'py> ResourceCostStore<'py> { - /// Connect when the adapter keeps resource costs and the target has a schema. - pub(crate) fn open(finalization: &Bound<'py, PyAny>) -> PyResult> { - let adapter: Bound<'py, PyAny> = native_field(finalization, "adapter")?; - if !adapter.hasattr(CREATE_TABLE_METHOD)? { - return Ok(None); - } - let Some(schema) = - native_field(finalization, "target_schema")?.extract::>()? - else { - return Ok(None); - }; - let location = Location { - database: native_field(finalization, "target_database")?.extract()?, - schema, - }; - let Some(table) = table(&adapter, &location)? else { - return Ok(None); - }; - let connection: Bound<'py, PyAny> = native_field(&adapter, "connect")? - .call1((native_field(finalization, "connection_config")?,))?; - Ok(Some(Self { - finalization: finalization.clone(), - adapter, - connection, - location, - table, - })) - } - - /// Create the schema and table if missing, then append one collection attempt. - pub(crate) fn append( - &self, - run: (&CostRunIdentity<'_>, i64), - summary: &CostSummary, - ) -> PyResult<()> { - let py: Python<'py> = self.adapter.py(); - let keywords = self.location.keywords(py); - for statement in call_with( - &native_field(&self.adapter, "render_create_schema")?, - &keywords, - )? - .try_iter()? - { - self.execute(&statement?.extract::()?)?; - } - let create: String = call_with( - &native_field(&self.adapter, CREATE_TABLE_METHOD)?, - &keywords, - )? - .extract()?; - if !create.is_empty() { - self.execute(&create)?; - } - let observed_at: String = call_with( - &native_field(&self.adapter, "render_current_utc_timestamp")?, - &[], - )? - .extract()?; - let columns: Vec<&str> = RESOURCE_COST_COLUMN_TYPES - .iter() - .map(|(column, _)| *column) - .collect(); - let cells: Vec> = observation_cells(py, run, summary, &observed_at)?; - for chunk in cells.chunks(ROWS_PER_STATEMENT) { - self.execute(&values_insert_sql(&self.table, &columns, chunk))?; - } - Ok(()) - } - - /// The newest run row of each refreshable run; none when the table is missing. - pub(crate) fn latest(&self) -> PyResult> { - let py: Python<'py> = self.adapter.py(); - let mut keywords = self.location.keywords(py); - keywords.push(("connection", self.connection.clone())); - keywords.push(( - "name", - PyString::new(py, RESOURCE_COSTS_TABLE_NAME).into_any(), - )); - if !call_with(&native_field(&self.adapter, "get_columns")?, &keywords)?.is_truthy()? { - return Ok(Vec::new()); - } - let result: Bound<'py, PyAny> = self.execute(&latest_run_costs_sql(&self.table))?; - call_with(&native_field(&result, "fetchall")?, &[])? - .try_iter()? - .map(|row| run_attempt(&row?)) - .collect() - } - - /// Re-collect one older run through the adapter, priced at `rate`. - pub(crate) fn recollect( - &self, - run: &RunCostAttempt, - (usd_per_credit, rate_source): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - ) -> PyResult> { - let py: Python<'py> = self.adapter.py(); - let parse = imported(py, "datetime", "datetime")?.getattr("fromisoformat")?; - let ledger: Bound<'py, PyAny> = native_field(&self.finalization, "project_dir")? - .call_method1("__truediv__", ("target",))? - .call_method1("__truediv__", ("executions",))? - .call_method1("__truediv__", (run.run_id.as_str(),))? - .call_method1("__truediv__", ("statements.jsonl",))?; - call_with( - &native_field(&self.adapter, "collect_run_cost")?, - &[ - ( - "connection_config", - native_field(&self.finalization, "connection_config")?, - ), - ( - "target_database", - native_field(&self.finalization, "target_database")?, - ), - ("run_id", PyString::new(py, &run.run_id).into_any()), - ("started_at", parse.call1((run.run_started_at.as_str(),))?), - ( - "completed_at", - parse.call1((run.run_completed_at.as_str(),))?, - ), - ("statement_ledger_path", ledger), - ("usd_per_credit", usd_per_credit.clone()), - ("rate_source", rate_source.clone()), - ( - "budget_seconds", - PyFloat::new(py, REFRESH_BUDGET_SECONDS).into_any(), - ), - ], - ) - } - - pub(crate) fn close(self) -> PyResult<()> { - native_field(&self.adapter, "close")? - .call1((&self.connection,)) - .map(|_| ()) - } - - fn execute(&self, sql: &str) -> PyResult> { - execute(&self.adapter, &self.connection, sql) - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!( - build_resource_costs_create_table_sql, - module - )?)?; - module.add_function(wrap_pyfunction!(build_resource_costs_prune_sql, module)?) -} diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/mod.rs b/crates/sqlbuild-python-front/src/resource_costs/main/mod.rs new file mode 100644 index 0000000000..14fe99725c --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/main/mod.rs @@ -0,0 +1 @@ +pub mod register_resource_costs_state; diff --git a/crates/sqlbuild-python-front/src/resource_costs/main/register_resource_costs_state.rs b/crates/sqlbuild-python-front/src/resource_costs/main/register_resource_costs_state.rs new file mode 100644 index 0000000000..502ef0673c --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/main/register_resource_costs_state.rs @@ -0,0 +1,15 @@ +//! Register the resource costs state functions and classes. + +use crate::resource_costs::_helpers::resource_costs_state::build_resource_costs_create_table_sql; +use crate::resource_costs::_helpers::resource_costs_state::build_resource_costs_prune_sql; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the resource costs state functions and classes. +pub fn register_resource_costs_state(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_resource_costs_create_table_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(build_resource_costs_prune_sql, module)?) +} diff --git a/crates/sqlbuild-python-front/src/resource_costs/mod.rs b/crates/sqlbuild-python-front/src/resource_costs/mod.rs new file mode 100644 index 0000000000..0ca49ee940 --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/mod.rs @@ -0,0 +1,5 @@ +//! The display-only resource costs table: its DDL, appends, latest reads and pruning. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-front/src/resource_costs/models.rs b/crates/sqlbuild-python-front/src/resource_costs/models.rs new file mode 100644 index 0000000000..6fb02bc785 --- /dev/null +++ b/crates/sqlbuild-python-front/src/resource_costs/models.rs @@ -0,0 +1,183 @@ +//! Public records of resource costs. + +use crate::native_host::constants::ROWS_PER_STATEMENT; +use crate::native_host::main::execute::execute; +use crate::native_host::models::Location; +use crate::resource_costs::_helpers::resource_costs_state::CREATE_TABLE_METHOD; +use crate::resource_costs::_helpers::resource_costs_state::REFRESH_BUDGET_SECONDS; +use crate::resource_costs::_helpers::resource_costs_state::observation_cells; +use crate::resource_costs::_helpers::resource_costs_state::run_attempt; +use crate::resource_costs::_helpers::resource_costs_state::table; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::types::PyFloat; +use pyo3::types::PyString; +use sqlbuild_core::constants::RESOURCE_COSTS_TABLE_NAME; +use sqlbuild_core::cost_report::models::CostSummary; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_state::resource_costs::constants::RESOURCE_COST_COLUMN_TYPES; +use sqlbuild_state::resource_costs::main::latest_run_costs_sql::latest_run_costs_sql; +use sqlbuild_state::resource_costs::models::RunCostAttempt; +use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; + +/// The run a collection belongs to, as the run row records it. +#[derive(Debug)] +pub struct CostRunIdentity<'a> { + pub run_id: &'a str, + pub orchestration_run_id: Option<&'a str>, + pub target_name: Option<&'a str>, + pub started_at: &'a str, + pub completed_at: &'a str, +} + +/// A target's resource costs table, open on one connection for one build's publication. +#[derive(Debug)] +pub struct ResourceCostStore<'py> { + pub(crate) finalization: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) connection: Bound<'py, PyAny>, + pub(crate) location: Location, + pub(crate) table: String, +} + +impl<'py> ResourceCostStore<'py> { + /// Connect when the adapter keeps resource costs and the target has a schema. + pub fn open(finalization: &Bound<'py, PyAny>) -> PyResult> { + let adapter: Bound<'py, PyAny> = native_field(finalization, "adapter")?; + if !adapter.hasattr(CREATE_TABLE_METHOD)? { + return Ok(None); + } + let Some(schema) = + native_field(finalization, "target_schema")?.extract::>()? + else { + return Ok(None); + }; + let location = Location { + database: native_field(finalization, "target_database")?.extract()?, + schema, + }; + let Some(table) = table(&adapter, &location)? else { + return Ok(None); + }; + let connection: Bound<'py, PyAny> = native_field(&adapter, "connect")? + .call1((native_field(finalization, "connection_config")?,))?; + Ok(Some(Self { + finalization: finalization.clone(), + adapter, + connection, + location, + table, + })) + } + + /// Create the schema and table if missing, then append one collection attempt. + pub fn append(&self, run: (&CostRunIdentity<'_>, i64), summary: &CostSummary) -> PyResult<()> { + let py: Python<'py> = self.adapter.py(); + let keywords = self.location.keywords(py); + for statement in call_with( + &native_field(&self.adapter, "render_create_schema")?, + &keywords, + )? + .try_iter()? + { + self.execute(&statement?.extract::()?)?; + } + let create: String = call_with( + &native_field(&self.adapter, CREATE_TABLE_METHOD)?, + &keywords, + )? + .extract()?; + if !create.is_empty() { + self.execute(&create)?; + } + let observed_at: String = call_with( + &native_field(&self.adapter, "render_current_utc_timestamp")?, + &[], + )? + .extract()?; + let columns: Vec<&str> = RESOURCE_COST_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect(); + let cells: Vec> = observation_cells(py, run, summary, &observed_at)?; + for chunk in cells.chunks(ROWS_PER_STATEMENT) { + self.execute(&values_insert_sql(&self.table, &columns, chunk))?; + } + Ok(()) + } + + /// The newest run row of each refreshable run; none when the table is missing. + pub fn latest(&self) -> PyResult> { + let py: Python<'py> = self.adapter.py(); + let mut keywords = self.location.keywords(py); + keywords.push(("connection", self.connection.clone())); + keywords.push(( + "name", + PyString::new(py, RESOURCE_COSTS_TABLE_NAME).into_any(), + )); + if !call_with(&native_field(&self.adapter, "get_columns")?, &keywords)?.is_truthy()? { + return Ok(Vec::new()); + } + let result: Bound<'py, PyAny> = self.execute(&latest_run_costs_sql(&self.table))?; + call_with(&native_field(&result, "fetchall")?, &[])? + .try_iter()? + .map(|row| run_attempt(&row?)) + .collect() + } + + /// Re-collect one older run through the adapter, priced at `rate`. + pub fn recollect( + &self, + run: &RunCostAttempt, + (usd_per_credit, rate_source): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + ) -> PyResult> { + let py: Python<'py> = self.adapter.py(); + let parse = imported(py, "datetime", "datetime")?.getattr("fromisoformat")?; + let ledger: Bound<'py, PyAny> = native_field(&self.finalization, "project_dir")? + .call_method1("__truediv__", ("target",))? + .call_method1("__truediv__", ("executions",))? + .call_method1("__truediv__", (run.run_id.as_str(),))? + .call_method1("__truediv__", ("statements.jsonl",))?; + call_with( + &native_field(&self.adapter, "collect_run_cost")?, + &[ + ( + "connection_config", + native_field(&self.finalization, "connection_config")?, + ), + ( + "target_database", + native_field(&self.finalization, "target_database")?, + ), + ("run_id", PyString::new(py, &run.run_id).into_any()), + ("started_at", parse.call1((run.run_started_at.as_str(),))?), + ( + "completed_at", + parse.call1((run.run_completed_at.as_str(),))?, + ), + ("statement_ledger_path", ledger), + ("usd_per_credit", usd_per_credit.clone()), + ("rate_source", rate_source.clone()), + ( + "budget_seconds", + PyFloat::new(py, REFRESH_BUDGET_SECONDS).into_any(), + ), + ], + ) + } + + pub fn close(self) -> PyResult<()> { + native_field(&self.adapter, "close")? + .call1((&self.connection,)) + .map(|_| ()) + } + + pub(crate) fn execute(&self, sql: &str) -> PyResult> { + execute(&self.adapter, &self.connection, sql) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index a8e12a58bc..1da6cc462f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -30,7 +30,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { freshness::registration::register(module)?; sqlbuild_python_front::native_host::main::register_consumption_state::register_consumption_state(module)?; sqlbuild_python_front::native_host::main::register_observations_state::register_observations_state(module)?; - crate::bindings::_helpers::native_host::resource_costs_state::register(module)?; + sqlbuild_python_front::resource_costs::main::register_resource_costs_state::register_resource_costs_state(module)?; sqlbuild_python_front::native_host::main::register_audit_results_state::register_audit_results_state(module)?; clone::registration::register(module)?; compute_logs::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs index c9767dedc0..34a11bf9c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/publication.rs @@ -12,10 +12,9 @@ use sqlbuild_state::resource_costs::main::refresh_candidates::refresh_candidates use sqlbuild_state::resource_costs::models::RunCostAttempt; use crate::bindings::_helpers::cost_report::records::summary; -use crate::bindings::_helpers::native_host::resource_costs_state::{ - CostRunIdentity, ResourceCostStore, -}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::resource_costs::models::CostRunIdentity; +use sqlbuild_python_front::resource_costs::models::ResourceCostStore; /// The attempt number of a build's own collection; each accepted re-read appends the next. const FIRST_ATTEMPT: i64 = 1; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/mod.rs index 164cfbcba9..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/mod.rs @@ -1,8 +1 @@ -pub(crate) mod appends; -pub(crate) mod conversion; -pub(crate) mod functions; -pub(crate) mod getters; -pub(crate) mod postgres; pub(crate) mod registration; -pub(crate) mod sqlite; -pub(crate) mod statements; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs deleted file mode 100644 index be8a2411f6..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs +++ /dev/null @@ -1,993 +0,0 @@ -//! `PostgresExecutionHistory`: deployed PostgreSQL event-log and run-projection storage. - -use sqlbuild_python_base::boundary::main::optional_text::optional_text; -use std::sync::OnceLock; -use std::sync::atomic::{AtomicBool, AtomicI64, Ordering}; - -use pyo3::IntoPyObjectExt; -use pyo3::exceptions::PyException; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyclass, pymethods}; -use pyo3::types::{PyBool, PyDateTime, PyDict, PyTuple}; -use sqlbuild_host::entries::models::ExecutorHost; -use sqlbuild_state::execution_history::constants::{ - DEFAULT_PAGE_LIMIT, POSTGRES_ADVISORY_LOCK, POSTGRES_ALL_EVENTS, POSTGRES_ALL_RUNS, - POSTGRES_CONNECT_TIMEOUT_SECONDS, POSTGRES_DELETE_RUNS, POSTGRES_EVENT_BY_ID, - POSTGRES_EVENT_EXISTS, POSTGRES_HISTORY_LOCK_ID, POSTGRES_MIGRATION_LOCK_ID, - POSTGRES_MIGRATIONS_EXIST, POSTGRES_NAMESPACE, POSTGRES_PING, POSTGRES_RECORD_MIGRATION, - POSTGRES_RETRYABLE_SQLSTATES, POSTGRES_RUN_BY_ID, POSTGRES_RUN_EXISTS, POSTGRES_SCHEMA, - POSTGRES_SCHEMA_VERSION, POSTGRES_SESSION_TIMEOUT, POSTGRES_STATEMENT_TIMEOUT_MS, - POSTGRES_TRANSACTION_RETRIES, POSTGRES_TRANSACTION_TIMEOUT, RUN_CURSOR_TIMESTAMP, - SCHEMA_VERSION, UNKNOWN_EVENT_CURSOR, UNKNOWN_RUN_CURSOR, -}; -use sqlbuild_state::execution_history::errors::HistoryErrorKind; -use sqlbuild_state::execution_history::main::decoded_run_cursor::decoded_run_cursor; -use sqlbuild_state::execution_history::main::event_cursor::event_cursor; -use sqlbuild_state::execution_history::main::event_cursor_position::event_cursor_position; -use sqlbuild_state::execution_history::main::event_insert_query::event_insert_query; -use sqlbuild_state::execution_history::main::event_page_query::event_page_query; -use sqlbuild_state::execution_history::main::project_runs::project_runs; -use sqlbuild_state::execution_history::main::run_cursor::run_cursor; -use sqlbuild_state::execution_history::main::run_insert_query::run_insert_query; -use sqlbuild_state::execution_history::main::run_page_query::run_page_query; -use sqlbuild_state::execution_history::main::schema_migration_needed::schema_migration_needed; -use sqlbuild_state::execution_history::main::validate_page_limit::validate_page_limit; -use sqlbuild_state::execution_history::main::validate_postgres_settings::validate_postgres_settings; -use sqlbuild_state::execution_history::main::validate_schema_target::validate_schema_target; -use sqlbuild_state::execution_history::main::validate_stored_content::validate_stored_content; -use sqlbuild_state::execution_history::models::{ - Dialect, PostgresSettings, PreparedEvent, Query, RunCursorKey, RunRow, SqlParam, -}; -use sqlbuild_state::execution_history::types::UtcTime; - -use crate::bindings::_helpers::execution_history::appends::prepared_batch; -use crate::bindings::_helpers::execution_history::conversion::{ - checked, event_filter_spec, exact_int, history_error_of_kind, new_namespace, now, - nullable_text, parsed_status, projection_facts, rejection, run_filter_spec, run_records, - run_rows, storage_error, stored_event, utc, utc_time, -}; -use crate::bindings::_helpers::execution_history::statements::{ - column, execute, execute_plain, execute_query, fetch_all, fetch_one, tuple_of, -}; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::boundary::main::within::within; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -const PSYCOPG_MODULE: &str = "psycopg"; -const PSYCOPG_ROWS_MODULE: &str = "psycopg.rows"; -const CLOSED: &str = "PostgreSQL execution history storage is closed"; -const MISSING_DRIVER: &str = "PostgreSQL execution history requires psycopg 3; install sqlbuild[postgres] or 'psycopg[binary]>=3.2'"; -const CONNECT_FAILED: &str = "cannot connect to PostgreSQL execution history; verify the secret-resolved DSN, network access, and sqlbuild[postgres] installation"; -const READ_FAILED: &str = "PostgreSQL execution history read failed"; -const METADATA_INVALID: &str = "PostgreSQL execution history schema metadata is invalid"; -const NAMESPACE_MISSING: &str = "PostgreSQL execution history cursor namespace is missing"; -const STORAGE_ERROR: ExecutorHost = ExecutorHost::ExecutionHistoryStorageError; - -/// The `psycopg.Error` class. -fn driver_error_class(py: Python<'_>) -> PyResult> { - imported(py, PSYCOPG_MODULE, "Error") -} - -fn driver_failure(py: Python<'_>, error: &PyErr) -> PyResult { - Ok(error.is_instance(py, &driver_error_class(py)?)) -} - -/// Serialization failures and deadlocks retry while attempts remain. -fn retries_transaction(sqlstate: Option<&str>, attempts: i64, max_retries: i64) -> bool { - sqlstate.is_some_and(|state| POSTGRES_RETRYABLE_SQLSTATES.contains(&state)) - && attempts < max_retries -} - -/// `value` when it is an `int` that is not a `bool`. -fn storage_position(value: &Bound<'_, PyAny>, message: &str) -> PyResult { - exact_int(value).ok_or_else(|| storage_error(value.py(), message)) -} - -/// `value.astimezone(UTC)` when it is a `datetime`. -fn utc_datetime<'py>(value: &Bound<'py, PyAny>, message: &str) -> PyResult> { - let py: Python<'py> = value.py(); - if !value.is_instance_of::() { - return Err(storage_error(py, message)); - } - value.call_method1("astimezone", (utc(py)?,)) -} - -fn optional_utc(value: &Bound<'_, PyAny>) -> PyResult> { - if !value.is_instance_of::() { - return Ok(None); - } - utc_time(value).map(Some) -} - -fn text_column(row: &Bound<'_, PyAny>, name: &str) -> PyResult { - column(row, name)?.str()?.extract() -} - -/// One `sqlbuild_run_projection` row as a projected run. -fn stored_run(row: &Bound<'_, PyAny>) -> PyResult { - let py: Python<'_> = row.py(); - let created_at = utc_datetime( - &column(row, "created_at")?, - "PostgreSQL run timestamp is invalid", - )?; - let last_storage_order: i64 = storage_position( - &column(row, "last_event_storage_id")?, - "PostgreSQL run storage position is invalid", - )?; - let optional_str = |name: &str| -> PyResult> { - let value = column(row, name)?; - if value.is_none() { - return Ok(None); - } - Ok(Some(value.str()?.extract()?)) - }; - Ok(RunRow { - run_id: text_column(row, "run_id")?, - invocation_id: text_column(row, "invocation_id")?, - created_at: utc_time(&created_at)?, - status: parsed_status(py, &text_column(row, "status")?)?, - is_complete: column(row, "is_complete")?.is_truthy()?, - last_event_cursor: text_column(row, "last_event_cursor")?, - last_storage_order, - command: optional_str("command")?, - target: optional_str("target")?, - environment: optional_str("environment")?, - started_at: optional_utc(&column(row, "started_at")?)?, - ended_at: optional_utc(&column(row, "ended_at")?)?, - }) -} - -/// Deployed PostgreSQL implementation of event-log and run-projection contracts. -#[pyclass( - module = "sqlbuild._native", - name = "PostgresExecutionHistory", - frozen, - subclass, - dict -)] -pub(crate) struct PostgresExecutionHistory { - statement_timeout_ms: OnceLock, - max_transaction_retries: OnceLock, - closed: AtomicBool, - project_calls: AtomicI64, -} - -impl PostgresExecutionHistory { - fn ensure_open(&self, py: Python<'_>) -> PyResult<()> { - if self.closed.load(Ordering::SeqCst) { - return Err(storage_error(py, CLOSED)); - } - Ok(()) - } - - /// Run `action` in a transaction with the statement timeout, retrying serialization failures. - fn transactional<'py, T>( - &self, - slf: &Bound<'py, Self>, - action: impl Fn(&Bound<'py, PyAny>) -> PyResult, - failure_message: &str, - ) -> PyResult { - let py: Python<'py> = slf.py(); - let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; - let timeout: String = self - .statement_timeout_ms - .get() - .copied() - .unwrap_or(POSTGRES_STATEMENT_TIMEOUT_MS) - .to_string(); - let max_retries: i64 = self - .max_transaction_retries - .get() - .copied() - .unwrap_or(POSTGRES_TRANSACTION_RETRIES); - let mut attempts: i64 = 0; - loop { - let result: PyResult = within(&connection.call_method0("transaction")?, |_| { - within(&connection.call_method0("cursor")?, |cursor| { - execute( - cursor, - POSTGRES_TRANSACTION_TIMEOUT, - &[SqlParam::Text(timeout.clone())], - )?; - action(cursor) - }) - }); - let error: PyErr = match result { - Ok(value) => return Ok(value), - Err(error) => error, - }; - if history_error_of_kind(py, &error, STORAGE_ERROR)? || !driver_failure(py, &error)? { - return Err(error); - } - let sqlstate: Option = - optional_text(&native_field(error.value(py), "sqlstate")?)?; - if retries_transaction(sqlstate.as_deref(), attempts, max_retries) { - attempts += 1; - continue; - } - return Err(storage_error(py, failure_message)); - } - } - - /// `query` through a fresh cursor, with driver failures reported as read failures. - fn read_rows<'py>(slf: &Bound<'py, Self>, query: &Query) -> PyResult>> { - let py: Python<'py> = slf.py(); - let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; - let rows = within(&connection.call_method0("cursor")?, |cursor| { - execute_query(cursor, query)?; - fetch_all(cursor) - }); - match rows { - Err(error) if driver_failure(py, &error)? => Err(storage_error(py, READ_FAILED)), - rows => rows, - } - } - - fn with_cursor<'py, T>( - slf: &Bound<'py, Self>, - cursor: Option<&Bound<'py, PyAny>>, - body: impl FnOnce(&Bound<'py, PyAny>) -> PyResult, - ) -> PyResult { - if let Some(cursor) = cursor { - return body(cursor); - } - let owned: Bound<'py, PyAny> = native_field(slf, "_connection")?.call_method0("cursor")?; - let result: PyResult = body(&owned); - let closed = owned.call_method0("close"); - let value: T = result?; - closed?; - Ok(value) - } - - fn schema_version_in<'py>( - slf: &Bound<'py, Self>, - cursor: Option<&Bound<'py, PyAny>>, - ) -> PyResult { - let py: Python<'py> = slf.py(); - Self::with_cursor(slf, cursor, |active| { - let read = (|| -> PyResult { - execute_plain(active, POSTGRES_MIGRATIONS_EXIST)?; - let Some(exists) = fetch_one(active)? else { - return Ok(0); - }; - if exists.get_item("table_name")?.is_none() { - return Ok(0); - } - execute_plain(active, POSTGRES_SCHEMA_VERSION)?; - let row = fetch_one(active)?; - let value = row.map(|row| row.get_item("schema_version")).transpose()?; - value - .as_ref() - .and_then(exact_int) - .ok_or_else(|| storage_error(py, METADATA_INVALID)) - })(); - match read { - Err(error) if history_error_of_kind(py, &error, STORAGE_ERROR)? => Err(error), - Err(error) if error.is_instance_of::(py) => { - Err(storage_error(py, METADATA_INVALID)) - } - read => read, - } - }) - } - - fn namespace_in<'py>( - slf: &Bound<'py, Self>, - cursor: Option<&Bound<'py, PyAny>>, - ) -> PyResult { - let py: Python<'py> = slf.py(); - Self::with_cursor(slf, cursor, |active| { - execute_plain(active, POSTGRES_NAMESPACE)?; - let row = fetch_one(active)?; - let namespace = match row { - Some(row) => optional_text(&row.get_item("storage_namespace")?)?, - None => None, - }; - namespace.ok_or_else(|| storage_error(py, NAMESPACE_MISSING)) - }) - } - - fn stored_event<'py>(row: &Bound<'py, PyAny>, namespace: &str) -> PyResult> { - let py: Python<'py> = row.py(); - let storage_id: i64 = storage_position( - &column(row, "storage_id")?, - "PostgreSQL event storage position is invalid", - )?; - let received_at = utc_datetime( - &column(row, "received_at")?, - "PostgreSQL event timestamp is invalid", - )?; - let payload: Bound<'py, PyAny> = column(row, "payload_json")?; - if optional_text(&payload)?.is_none() { - return Err(storage_error( - py, - "PostgreSQL canonical event content is invalid", - )); - } - stored_event( - py, - ( - storage_id, - event_cursor(Dialect::Postgres, namespace, storage_id), - ), - received_at, - &payload, - ) - } - - fn stored_events<'py>( - slf: &Bound<'py, Self>, - rows: &[Bound<'py, PyAny>], - ) -> PyResult>> { - if rows.is_empty() { - return Ok(Vec::new()); - } - let namespace: String = Self::namespace_in(slf, None)?; - rows.iter() - .map(|row| Self::stored_event(row, &namespace)) - .collect() - } - - fn read_all_runs(cursor: &Bound<'_, PyAny>) -> PyResult> { - execute_plain(cursor, POSTGRES_ALL_RUNS)?; - fetch_all(cursor)?.iter().map(stored_run).collect() - } - - fn lock_history(cursor: &Bound<'_, PyAny>) -> PyResult<()> { - execute( - cursor, - POSTGRES_ADVISORY_LOCK, - &[SqlParam::Integer(POSTGRES_HISTORY_LOCK_ID)], - )?; - Ok(()) - } - - fn append_via<'py>( - slf: &Bound<'py, Self>, - cursor: &Bound<'py, PyAny>, - events: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - native_field(slf, "_append_events")?.call( - (), - Some(&keywords( - py, - &[("cursor", cursor.clone()), ("events", events.clone())], - )?), - ) - } - - fn replace_via<'py>( - slf: &Bound<'py, Self>, - cursor: &Bound<'py, PyAny>, - projected: &Bound<'py, PyAny>, - ) -> PyResult<()> { - let py: Python<'py> = slf.py(); - native_field(slf, "_replace_projection")?.call( - (), - Some(&keywords( - py, - &[("cursor", cursor.clone()), ("projected", projected.clone())], - )?), - )?; - Ok(()) - } - - fn migrate(&self, slf: &Bound<'_, Self>) -> PyResult<()> { - let py: Python<'_> = slf.py(); - self.transactional( - slf, - |cursor| { - execute( - cursor, - POSTGRES_ADVISORY_LOCK, - &[SqlParam::Integer(POSTGRES_MIGRATION_LOCK_ID)], - )?; - let current: i64 = Self::schema_version_in(slf, Some(cursor))?; - if !checked(py, schema_migration_needed(Dialect::Postgres, current))? { - return Ok(()); - } - for statement in POSTGRES_SCHEMA { - execute_plain(cursor, statement)?; - } - execute( - cursor, - POSTGRES_RECORD_MIGRATION, - &[ - SqlParam::Integer(SCHEMA_VERSION), - SqlParam::Text(new_namespace(py)?), - ], - )?; - Ok(()) - }, - "PostgreSQL execution history schema migration failed", - ) - } - - fn run_cursor_key( - slf: &Bound<'_, Self>, - cursor: Option<&Bound<'_, PyAny>>, - ) -> PyResult> { - let py: Python<'_> = slf.py(); - let Some(cursor) = cursor.filter(|cursor| !cursor.is_none()) else { - return Ok(None); - }; - let text: String = optional_text(cursor)?.unwrap_or_default(); - let namespace: String = Self::namespace_in(slf, None)?; - let decoded = checked(py, decoded_run_cursor(Dialect::Postgres, &text, &namespace))?; - let invalid = || rejection(py, HistoryErrorKind::InvalidCursor, RUN_CURSOR_TIMESTAMP); - let parsed = imported(py, "datetime", "datetime")? - .call_method1("fromisoformat", (decoded.created_at_text.as_str(),)); - let parsed = match parsed { - Ok(parsed) => parsed, - Err(error) if error.is_instance_of::(py) => { - return Err(invalid()); - } - Err(error) => return Err(error), - }; - if parsed.call_method0("utcoffset")?.is_none() { - return Err(invalid()); - } - let created_at = utc_time(&parsed)?; - let rows = Self::read_rows( - slf, - &Query { - sql: POSTGRES_RUN_EXISTS.to_owned(), - params: vec![ - SqlParam::Timestamp(created_at), - SqlParam::Text(decoded.run_id.clone()), - ], - }, - )?; - if rows.is_empty() { - return Err(rejection( - py, - HistoryErrorKind::InvalidCursor, - UNKNOWN_RUN_CURSOR, - )); - } - Ok(Some(RunCursorKey { - created_at, - created_at_text: decoded.created_at_text, - run_id: decoded.run_id, - })) - } -} - -#[pymethods] -impl PostgresExecutionHistory { - #[new] - #[pyo3(signature = (*_args, **_kwargs))] - fn new(_args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>) -> Self { - Self { - statement_timeout_ms: OnceLock::new(), - max_transaction_retries: OnceLock::new(), - closed: AtomicBool::new(true), - project_calls: AtomicI64::new(0), - } - } - - #[pyo3(signature = (dsn, **options))] - fn __init__( - slf: &Bound<'_, Self>, - dsn: &Bound<'_, PyAny>, - options: Option>, - ) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let this: &Self = slf.get(); - let options = KeywordArguments::read( - py, - "__init__", - options, - ( - &[], - &[ - "connect_timeout_seconds", - "statement_timeout_ms", - "max_transaction_retries", - ], - ), - )?; - let setting = |name: &str, default: i64| -> PyResult> { - if options.passed(name)? { - return Ok(exact_int(&options.value(name)?)); - } - Ok(Some(default)) - }; - let settings = PostgresSettings { - dsn: optional_text(dsn)?, - connect_timeout_seconds: setting( - "connect_timeout_seconds", - POSTGRES_CONNECT_TIMEOUT_SECONDS, - )?, - statement_timeout_ms: setting("statement_timeout_ms", POSTGRES_STATEMENT_TIMEOUT_MS)?, - max_transaction_retries: setting( - "max_transaction_retries", - POSTGRES_TRANSACTION_RETRIES, - )?, - }; - checked(py, validate_postgres_settings(&settings))?; - let (psycopg, dict_row) = match ( - py.import(PSYCOPG_MODULE), - imported(py, PSYCOPG_ROWS_MODULE, "dict_row"), - ) { - (Ok(psycopg), Ok(dict_row)) => (psycopg, dict_row), - (Err(error), _) | (_, Err(error)) - if error.is_instance_of::(py) => - { - return Err(storage_error(py, MISSING_DRIVER)); - } - (Err(error), _) | (_, Err(error)) => return Err(error), - }; - let statement_timeout: i64 = settings - .statement_timeout_ms - .unwrap_or(POSTGRES_STATEMENT_TIMEOUT_MS); - let _ = this.max_transaction_retries.set( - settings - .max_transaction_retries - .unwrap_or(POSTGRES_TRANSACTION_RETRIES), - ); - this.closed.store(false, Ordering::SeqCst); - let mut connection: Option> = None; - let opened: PyResult<()> = (|| { - let opened: Bound<'_, PyAny> = native_field(&psycopg, "connect")?.call( - (dsn,), - Some(&keywords( - py, - &[ - ( - "connect_timeout", - settings.connect_timeout_seconds.into_bound_py_any(py)?, - ), - ("autocommit", PyBool::new(py, true).to_owned().into_any()), - ("row_factory", dict_row.clone()), - ], - )?), - )?; - connection = Some(opened.clone()); - slf.setattr("_connection", &opened)?; - within(&opened.call_method0("cursor")?, |cursor| { - execute( - cursor, - POSTGRES_SESSION_TIMEOUT, - &[SqlParam::Text(statement_timeout.to_string())], - ) - })?; - let _ = this.statement_timeout_ms.set(statement_timeout); - this.migrate(slf)?; - slf.call_method0("reconcile")?; - Ok(()) - })(); - let Err(error) = opened else { - return Ok(()); - }; - if !error.is_instance_of::(py) { - return Err(error); - } - if let Some(connection) = connection { - let _ = connection.call_method0("close"); - } - this.closed.store(true, Ordering::SeqCst); - if history_error_of_kind(py, &error, STORAGE_ERROR)? { - return Err(error); - } - Err(storage_error(py, CONNECT_FAILED)) - } - - fn __repr__(slf: &Bound<'_, Self>) -> PyResult { - let state: &str = if slf.get().closed.load(Ordering::SeqCst) { - "closed" - } else { - "open" - }; - let name: String = slf.get_type().getattr("__name__")?.extract()?; - Ok(format!("{name}(state='{state}')")) - } - - /// The number of incremental projection calls. - #[getter] - pub(crate) fn project_calls(&self) -> i64 { - self.project_calls.load(Ordering::SeqCst) - } - - fn append_event<'py>( - slf: &Bound<'py, Self>, - event: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - slf.get().ensure_open(py)?; - slf.call_method1("append_events", (tuple_of(py, vec![event.clone()])?,))? - .get_item(0) - } - - fn append_events<'py>( - slf: &Bound<'py, Self>, - events: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - let this: &Self = slf.get(); - this.ensure_open(py)?; - let pending: Bound<'py, PyTuple> = - PyTuple::new(py, events.try_iter()?.collect::>>()?)?; - this.transactional( - slf, - |cursor| { - Self::lock_history(cursor)?; - Self::append_via(slf, cursor, pending.as_any()) - }, - "PostgreSQL event append failed", - ) - } - - #[pyo3(signature = (*, event_filter, after_cursor = None, limit = None))] - #[pyo3(name = "get_events")] - fn stored_events_page<'py>( - slf: &Bound<'py, Self>, - event_filter: &Bound<'py, PyAny>, - after_cursor: Option<&Bound<'py, PyAny>>, - limit: Option<&Bound<'py, PyAny>>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - slf.get().ensure_open(py)?; - let limit: Option = limit.map_or(Some(DEFAULT_PAGE_LIMIT), exact_int); - checked(py, validate_page_limit(limit))?; - let limit: i64 = limit.unwrap_or(DEFAULT_PAGE_LIMIT); - let after: i64 = match after_cursor.filter(|cursor| !cursor.is_none()) { - None => 0, - Some(cursor) => { - let text: String = optional_text(cursor)?.unwrap_or_default(); - let namespace: String = Self::namespace_in(slf, None)?; - let position: i64 = checked( - py, - event_cursor_position(Dialect::Postgres, Some(&text), &namespace), - )?; - let rows = Self::read_rows( - slf, - &Query { - sql: POSTGRES_EVENT_EXISTS.to_owned(), - params: vec![SqlParam::Integer(position)], - }, - )?; - if rows.is_empty() { - return Err(rejection( - py, - HistoryErrorKind::InvalidCursor, - UNKNOWN_EVENT_CURSOR, - )); - } - position - } - }; - let spec = event_filter_spec(event_filter)?; - let rows = Self::read_rows( - slf, - &event_page_query(Dialect::Postgres, after, &spec, limit), - )?; - let has_more: bool = rows.len() as i64 > limit; - let page: Vec> = rows.into_iter().take(limit as usize).collect(); - let records: Vec> = Self::stored_events(slf, &page)?; - let next_cursor: Bound<'py, PyAny> = match records.last() { - Some(record) => native_field(record, "cursor")?, - None => py.None().into_bound(py), - }; - call_with( - &host_entry(py, ExecutorHost::EventPage)?, - &[ - ("records", tuple_of(py, records)?.into_any()), - ("next_cursor", next_cursor), - ("has_more", has_more.into_bound_py_any(py)?), - ], - ) - } - - #[pyo3(name = "get_run")] - fn run_record<'py>( - slf: &Bound<'py, Self>, - run_id: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - slf.get().ensure_open(py)?; - let run_id: Option = nullable_text(run_id)?; - let params: Vec = vec![run_id.map_or(SqlParam::Null, SqlParam::Text)]; - let rows = Self::read_rows( - slf, - &Query { - sql: POSTGRES_RUN_BY_ID.to_owned(), - params, - }, - )?; - match rows.first() { - Some(row) => crate::bindings::_helpers::execution_history::conversion::run_record( - py, - &stored_run(row)?, - ), - None => Ok(py.None().into_bound(py)), - } - } - - #[pyo3(signature = (*, run_filter, after_cursor = None, limit = None))] - #[pyo3(name = "get_runs")] - fn run_page<'py>( - slf: &Bound<'py, Self>, - run_filter: &Bound<'py, PyAny>, - after_cursor: Option<&Bound<'py, PyAny>>, - limit: Option<&Bound<'py, PyAny>>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - slf.get().ensure_open(py)?; - let limit: Option = limit.map_or(Some(DEFAULT_PAGE_LIMIT), exact_int); - checked(py, validate_page_limit(limit))?; - let limit: i64 = limit.unwrap_or(DEFAULT_PAGE_LIMIT); - let after = Self::run_cursor_key(slf, after_cursor)?; - let spec = run_filter_spec(run_filter)?; - let rows = Self::read_rows( - slf, - &run_page_query(Dialect::Postgres, after.as_ref(), &spec, limit), - )?; - let runs: Vec = rows - .iter() - .take(limit as usize) - .map(stored_run) - .collect::>()?; - let next_cursor: Bound<'py, PyAny> = match runs.last() { - Some(run) => { - let namespace: String = Self::namespace_in(slf, None)?; - checked(py, run_cursor(Dialect::Postgres, &namespace, run))? - .into_bound_py_any(py)? - } - None => py.None().into_bound(py), - }; - call_with( - &host_entry(py, ExecutorHost::RunPage)?, - &[ - ("records", run_records(py, &runs)?.into_any()), - ("next_cursor", next_cursor), - ( - "has_more", - (rows.len() as i64 > limit).into_bound_py_any(py)?, - ), - ], - ) - } - - fn project<'py>( - slf: &Bound<'py, Self>, - stored_events: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - let this: &Self = slf.get(); - this.ensure_open(py)?; - let pending: Bound<'py, PyTuple> = - PyTuple::new(py, stored_events.try_iter()?.collect::>>()?)?; - this.project_calls.fetch_add(1, Ordering::SeqCst); - this.transactional( - slf, - |cursor| { - Self::lock_history(cursor)?; - let facts = projection_facts(pending.as_any())?; - let projected = checked(py, project_runs(facts, Self::read_all_runs(cursor)?))?; - let records: Bound<'py, PyTuple> = run_records(py, &projected)?; - Self::replace_via(slf, cursor, records.as_any())?; - Ok(records) - }, - "PostgreSQL run projection publication failed", - ) - } - - fn rebuild_from_events<'py>( - slf: &Bound<'py, Self>, - stored_events: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - let this: &Self = slf.get(); - this.ensure_open(py)?; - let pending: Bound<'py, PyTuple> = - PyTuple::new(py, stored_events.try_iter()?.collect::>>()?)?; - this.transactional( - slf, - |cursor| { - Self::lock_history(cursor)?; - let projected = checked( - py, - project_runs(projection_facts(pending.as_any())?, Vec::new()), - )?; - let records: Bound<'py, PyTuple> = run_records(py, &projected)?; - Self::replace_via(slf, cursor, records.as_any())?; - Ok(records) - }, - "PostgreSQL run projection rebuild failed", - ) - } - - /// Atomically append immutable facts and publish their run projection. - fn append_and_project<'py>( - slf: &Bound<'py, Self>, - events: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - let this: &Self = slf.get(); - this.ensure_open(py)?; - let pending: Bound<'py, PyTuple> = - PyTuple::new(py, events.try_iter()?.collect::>>()?)?; - this.transactional( - slf, - |cursor| { - Self::lock_history(cursor)?; - let stored: Bound<'py, PyAny> = Self::append_via(slf, cursor, pending.as_any())?; - let projected = checked( - py, - project_runs(projection_facts(&stored)?, Self::read_all_runs(cursor)?), - )?; - let records: Bound<'py, PyTuple> = run_records(py, &projected)?; - Self::replace_via(slf, cursor, records.as_any())?; - Ok(stored) - }, - "PostgreSQL atomic append and projection failed", - ) - } - - /// Rebuild the disposable run projection from all durable event facts. - fn reconcile<'py>(slf: &Bound<'py, Self>) -> PyResult> { - let py: Python<'py> = slf.py(); - let this: &Self = slf.get(); - this.ensure_open(py)?; - this.transactional( - slf, - |cursor| { - Self::lock_history(cursor)?; - let stored: Bound<'py, PyAny> = native_field(slf, "_read_all_events")? - .call((), Some(&keywords(py, &[("cursor", cursor.clone())])?))?; - let projected = checked(py, project_runs(projection_facts(&stored)?, Vec::new()))?; - let records: Bound<'py, PyTuple> = run_records(py, &projected)?; - Self::replace_via(slf, cursor, records.as_any())?; - Ok(records) - }, - "PostgreSQL reconciliation failed", - ) - } - - /// Verify connectivity and the supported schema revision. - fn check_health(slf: &Bound<'_, Self>) -> PyResult { - let py: Python<'_> = slf.py(); - slf.get().ensure_open(py)?; - let healthy = (|| -> PyResult { - let connection: Bound<'_, PyAny> = native_field(slf, "_connection")?; - let row = within(&connection.call_method0("cursor")?, |cursor| { - execute_plain(cursor, POSTGRES_PING)?; - fetch_one(cursor) - })?; - let version: i64 = slf.call_method0("get_schema_version")?.extract()?; - Ok(row.is_some() && version == SCHEMA_VERSION) - })(); - match healthy { - Err(error) if driver_failure(py, &error)? => Ok(false), - healthy => healthy, - } - } - - #[pyo3(name = "get_schema_version")] - fn schema_version(slf: &Bound<'_, Self>) -> PyResult { - slf.get().ensure_open(slf.py())?; - Self::schema_version_in(slf, None) - } - - #[pyo3(signature = (*, target_version = None))] - fn upgrade_schema(slf: &Bound<'_, Self>, target_version: Option) -> PyResult { - let py: Python<'_> = slf.py(); - let this: &Self = slf.get(); - this.ensure_open(py)?; - let target: i64 = target_version.unwrap_or(SCHEMA_VERSION); - checked(py, validate_schema_target(Dialect::Postgres, target))?; - this.migrate(slf)?; - Ok(target) - } - - fn close(slf: &Bound<'_, Self>) -> PyResult<()> { - let this: &Self = slf.get(); - if this.closed.load(Ordering::SeqCst) { - return Ok(()); - } - native_field(slf, "_connection")?.call_method0("close")?; - this.closed.store(true, Ordering::SeqCst); - Ok(()) - } - - fn dispose(slf: &Bound<'_, Self>) -> PyResult<()> { - slf.call_method0("close")?; - Ok(()) - } - - fn __enter__<'py>(slf: &Bound<'py, Self>) -> PyResult> { - slf.get().ensure_open(slf.py())?; - Ok(slf.clone()) - } - - #[pyo3(signature = (*_exc_info))] - fn __exit__(slf: &Bound<'_, Self>, _exc_info: &Bound<'_, PyTuple>) -> PyResult> { - slf.call_method0("close")?; - Ok(None) - } - - #[pyo3(signature = (*, cursor, events))] - fn _append_events<'py>( - slf: &Bound<'py, Self>, - cursor: &Bound<'py, PyAny>, - events: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - let prepared: Vec = prepared_batch(events)?; - let mut stored: Vec> = Vec::with_capacity(prepared.len()); - let mut namespace: Option = None; - let received_at = now(py)?; - for event in &prepared { - execute_query( - cursor, - &event_insert_query(Dialect::Postgres, event, &received_at), - )?; - let mut row = fetch_one(cursor)?; - if row.is_none() { - execute( - cursor, - POSTGRES_EVENT_BY_ID, - &[SqlParam::Text(event.event_id.clone())], - )?; - row = fetch_one(cursor)?; - } - let Some(row) = row else { - return Err(storage_error( - py, - "PostgreSQL event append did not produce a durable fact", - )); - }; - let digest: Option = optional_text(&column(&row, "content_digest")?)?; - let payload: Option = optional_text(&column(&row, "payload_json")?)?; - checked( - py, - validate_stored_content(event, (digest.as_deref(), payload.as_deref())), - )?; - if namespace.is_none() { - namespace = Some(Self::namespace_in(slf, Some(cursor))?); - } - stored.push(Self::stored_event( - &row, - namespace.as_deref().unwrap_or_default(), - )?); - } - tuple_of(py, stored) - } - - #[pyo3(signature = (*, cursor, projected))] - fn _replace_projection( - _slf: &Bound<'_, Self>, - cursor: &Bound<'_, PyAny>, - projected: &Bound<'_, PyAny>, - ) -> PyResult<()> { - execute_plain(cursor, POSTGRES_DELETE_RUNS)?; - for run in run_rows(projected)? { - execute_query(cursor, &run_insert_query(Dialect::Postgres, &run))?; - } - Ok(()) - } - - #[pyo3(signature = (*, cursor))] - fn _read_all_events<'py>( - slf: &Bound<'py, Self>, - cursor: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - execute_plain(cursor, POSTGRES_ALL_EVENTS)?; - let rows: Vec> = fetch_all(cursor)?; - let stored: Vec> = if rows.is_empty() { - Vec::new() - } else { - let namespace: String = Self::namespace_in(slf, Some(cursor))?; - rows.iter() - .map(|row| Self::stored_event(row, &namespace)) - .collect::>()? - }; - tuple_of(py, stored) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/registration.rs index 52d85a14b2..1d07cb8c4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/registration.rs @@ -2,12 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use crate::bindings::_helpers::execution_history::functions; -use crate::bindings::_helpers::execution_history::postgres::PostgresExecutionHistory; -use crate::bindings::_helpers::execution_history::sqlite::SqliteExecutionHistory; +use sqlbuild_python_front::execution_history::main::register_functions::register_functions; +use sqlbuild_python_front::execution_history::models::PostgresExecutionHistory; +use sqlbuild_python_front::execution_history::models::SqliteExecutionHistory; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_class::()?; - functions::register(module) + register_functions(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs deleted file mode 100644 index 509575ace2..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs +++ /dev/null @@ -1,945 +0,0 @@ -//! `SQLiteExecutionHistory`: project-local SQLite event-log and run-projection storage. - -use sqlbuild_python_base::boundary::main::optional_text::optional_text; -use std::fs::DirBuilder; -use std::io::ErrorKind; -use std::path::Path; -use std::sync::OnceLock; -use std::sync::atomic::{AtomicBool, AtomicI64, Ordering}; - -use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyclass, pymethods}; -use pyo3::types::{PyDict, PyTuple}; -use sqlbuild_host::entries::models::ExecutorHost; -use sqlbuild_state::execution_history::constants::{ - DEFAULT_PAGE_LIMIT, MEMORY_PATH, RUN_CURSOR_TIMESTAMP, SCHEMA_VERSION, SQLITE_ALL_EVENTS, - SQLITE_ALL_RUNS, SQLITE_BEGIN, SQLITE_BUSY_TIMEOUT_MS, SQLITE_COMMIT, SQLITE_DELETE_RUNS, - SQLITE_EVENT_BY_ID, SQLITE_EVENT_BY_STORAGE_ID, SQLITE_EVENT_EXISTS, SQLITE_FOREIGN_KEYS, - SQLITE_HEALTHY_CHECK_RESULT, SQLITE_INSERT_NAMESPACE, SQLITE_INSERT_SCHEMA_VERSION, - SQLITE_JOURNAL_MODE, SQLITE_METADATA_EXISTS, SQLITE_METADATA_TABLE, SQLITE_METADATA_VALUE, - SQLITE_NAMESPACE_KEY, SQLITE_QUICK_CHECK, SQLITE_ROLLBACK, SQLITE_RUN_BY_ID, SQLITE_RUN_EXISTS, - SQLITE_RUN_INSERT, SQLITE_SCHEMA, SQLITE_SCHEMA_VERSION_KEY, UNKNOWN_EVENT_CURSOR, - UNKNOWN_RUN_CURSOR, -}; -use sqlbuild_state::execution_history::errors::HistoryErrorKind; -use sqlbuild_state::execution_history::main::decoded_run_cursor::decoded_run_cursor; -use sqlbuild_state::execution_history::main::event_cursor::event_cursor; -use sqlbuild_state::execution_history::main::event_cursor_position::event_cursor_position; -use sqlbuild_state::execution_history::main::event_insert_query::event_insert_query; -use sqlbuild_state::execution_history::main::event_page_query::event_page_query; -use sqlbuild_state::execution_history::main::project_runs::project_runs; -use sqlbuild_state::execution_history::main::run_cursor::run_cursor; -use sqlbuild_state::execution_history::main::run_insert_query::run_insert_query; -use sqlbuild_state::execution_history::main::run_page_query::run_page_query; -use sqlbuild_state::execution_history::main::schema_migration_needed::schema_migration_needed; -use sqlbuild_state::execution_history::main::sqlite_database_path::sqlite_database_path; -use sqlbuild_state::execution_history::main::validate_page_limit::validate_page_limit; -use sqlbuild_state::execution_history::main::validate_schema_target::validate_schema_target; -use sqlbuild_state::execution_history::main::validate_stored_content::validate_stored_content; -use sqlbuild_state::execution_history::models::{ - Dialect, PreparedEvent, RunCursorKey, RunRow, SqlParam, SqliteLocation, -}; -use sqlbuild_state::execution_history::types::UtcTime; - -use crate::bindings::_helpers::execution_history::appends::prepared_batch; -use crate::bindings::_helpers::execution_history::conversion::{ - checked, event_filter_spec, exact_int, history_error_of_kind, new_namespace, now, - nullable_text, parsed_iso, parsed_status, projection_facts, python_params, rejection, - run_filter_spec, run_records, run_rows, storage_error, storage_error_from, stored_event, - utc_time, -}; -use crate::bindings::_helpers::execution_history::statements::{ - column, execute, execute_plain, execute_query, fetch_one, tuple_of, -}; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::native_threads::models::ReentrantLock; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::warehouse_access::main::sqlite_params::sqlite_params; -use sqlbuild_python_base::warehouse_access::models::SqliteConnection; -use sqlbuild_python_base::warehouse_access::models::SqliteCursor; - -const SQLITE_MODULE: &str = "sqlite3"; -const PATHLIB_MODULE: &str = "pathlib"; -const CLOSED: &str = "SQLite execution history storage is closed"; -const NOT_AWARE: &str = "SQLite history timestamp is not timezone-aware"; -const INTEGRITY_CONFLICT: ExecutorHost = ExecutorHost::IntegrityConflictError; -const STORAGE_ERROR: ExecutorHost = ExecutorHost::ExecutionHistoryStorageError; - -/// `Path(path).parent.mkdir(mode=0o700, parents=True, exist_ok=True)` for a file-backed history. -fn prepare_history_directory(path: &str) -> std::io::Result<()> { - if path == MEMORY_PATH { - return Ok(()); - } - let Some(parent) = Path::new(path).parent() else { - return Ok(()); - }; - if parent.as_os_str().is_empty() || parent.is_dir() { - return Ok(()); - } - if let Some(grandparent) = parent.parent() - && !grandparent.as_os_str().is_empty() - { - std::fs::create_dir_all(grandparent)?; - } - match private_directory_builder().create(parent) { - Err(error) if error.kind() == ErrorKind::AlreadyExists && parent.is_dir() => Ok(()), - result => result, - } -} - -/// A directory builder with the owner-only mode Python's `os.mkdir(mode=0o700)` applies. -#[cfg(unix)] -fn private_directory_builder() -> DirBuilder { - use std::os::unix::fs::DirBuilderExt; - let mut builder: DirBuilder = DirBuilder::new(); - builder.mode(sqlbuild_state::execution_history::constants::SQLITE_DIRECTORY_MODE); - builder -} - -/// Windows ignores the directory mode, as `os.mkdir` does there. -#[cfg(not(unix))] -fn private_directory_builder() -> DirBuilder { - DirBuilder::new() -} - -/// `os.chmod(path, 0o600)` for a file-backed history. -fn restrict_history_file(path: &str) -> std::io::Result<()> { - if path == MEMORY_PATH { - return Ok(()); - } - set_owner_only(path) -} - -#[cfg(unix)] -fn set_owner_only(path: &str) -> std::io::Result<()> { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions( - path, - std::fs::Permissions::from_mode( - sqlbuild_state::execution_history::constants::SQLITE_FILE_MODE, - ), - ) -} - -/// `os.chmod(path, 0o600)` on Windows only keeps the file writable, which it already is. -#[cfg(not(unix))] -fn set_owner_only(_path: &str) -> std::io::Result<()> { - Ok(()) -} - -/// Whether `error` is a `sqlite3.Error`. -fn sqlite_failure(py: Python<'_>, error: &PyErr) -> PyResult { - Ok(error.is_instance(py, &imported(py, SQLITE_MODULE, "Error")?)) -} - -/// `result`, with a `sqlite3.Error` raised as a storage error `message` from it. -fn wrapped(py: Python<'_>, result: PyResult, message: &str) -> PyResult { - match result { - Err(error) if sqlite_failure(py, &error)? => Err(storage_error_from(py, message, error)), - result => result, - } -} - -/// A stored ISO timestamp as an aware UTC `datetime`. -fn parsed_timestamp<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = value.py(); - let text: String = value.extract()?; - let parsed: Bound<'py, PyAny> = imported(py, "datetime", "datetime")? - .call_method1("fromisoformat", (text.replace('Z', "+00:00"),))?; - if parsed.call_method0("utcoffset")?.is_none() { - return Err(storage_error(py, NOT_AWARE)); - } - parsed.call_method1("astimezone", (imported(py, "datetime", "UTC")?,)) -} - -fn optional_timestamp(value: &Bound<'_, PyAny>) -> PyResult> { - if value.is_none() { - return Ok(None); - } - utc_time(&parsed_timestamp(value)?).map(Some) -} - -/// One `run_projection` row as a projected run. -fn stored_run(row: &Bound<'_, PyAny>) -> PyResult { - let py: Python<'_> = row.py(); - let status: String = column(row, "status")?.extract()?; - Ok(RunRow { - run_id: column(row, "run_id")?.extract()?, - invocation_id: column(row, "invocation_id")?.extract()?, - created_at: utc_time(&parsed_timestamp(&column(row, "created_at")?)?)?, - status: parsed_status(py, &status)?, - is_complete: column(row, "is_complete")?.is_truthy()?, - last_event_cursor: column(row, "last_event_cursor")?.extract()?, - last_storage_order: column(row, "last_storage_order")?.extract()?, - command: nullable_text(&column(row, "command")?)?, - target: nullable_text(&column(row, "target")?)?, - environment: nullable_text(&column(row, "environment")?)?, - started_at: optional_timestamp(&column(row, "started_at")?)?, - ended_at: optional_timestamp(&column(row, "ended_at")?)?, - }) -} - -/// Project-local SQLite implementation of event-log and run-projection storage contracts. -#[pyclass( - module = "sqlbuild._native", - name = "SQLiteExecutionHistory", - frozen, - subclass, - dict -)] -pub(crate) struct SqliteExecutionHistory { - path: OnceLock, - lock: ReentrantLock, - closed: AtomicBool, - project_calls: AtomicI64, -} - -impl SqliteExecutionHistory { - fn database_path(&self) -> &str { - self.path.get().map_or(MEMORY_PATH, String::as_str) - } - - fn with_lock<'py, T>( - &self, - py: Python<'py>, - body: impl FnOnce() -> PyResult, - ) -> PyResult { - self.lock.with_lock(py, body) - } - - fn ensure_open(&self, py: Python<'_>) -> PyResult<()> { - if self.closed.load(Ordering::SeqCst) { - return Err(storage_error(py, CLOSED)); - } - Ok(()) - } - - /// Run `body` in `BEGIN IMMEDIATE`, committing on success and rolling back on failure. - fn transaction<'py, T>( - &self, - slf: &Bound<'py, Self>, - body: impl FnOnce(&Bound<'py, PyAny>) -> PyResult, - ) -> PyResult { - let py: Python<'py> = slf.py(); - self.with_lock(py, || { - self.ensure_open(py)?; - let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; - execute_plain(&connection, SQLITE_BEGIN)?; - let value: T = match body(&connection) { - Ok(value) => value, - Err(error) => { - let _ = execute_plain(&connection, SQLITE_ROLLBACK); - return Err(error); - } - }; - if let Err(error) = execute_plain(&connection, SQLITE_COMMIT) { - if sqlite_failure(py, &error)? - && match connection.cast::() { - Ok(connection) => connection.get().in_transaction(py)?, - Err(_) => native_field(&connection, "in_transaction")?.is_truthy()?, - } - { - let _ = execute_plain(&connection, SQLITE_ROLLBACK); - } - return Err(error); - } - Ok(value) - }) - } - - fn metadata_value<'py>( - connection: &Bound<'py, PyAny>, - key: &str, - ) -> PyResult>> { - let row = fetch_one(&execute( - connection, - SQLITE_METADATA_VALUE, - &[SqlParam::Text(key.to_owned())], - )?)?; - row.map(|row| row.get_item(0)).transpose() - } - - fn metadata_version(connection: &Bound<'_, PyAny>) -> PyResult { - let py: Python<'_> = connection.py(); - let exists = fetch_one(&execute( - connection, - SQLITE_METADATA_EXISTS, - &[SqlParam::Text(SQLITE_METADATA_TABLE.to_owned())], - )?)?; - if exists.is_none() { - return Ok(0); - } - let Some(value) = Self::metadata_value(connection, SQLITE_SCHEMA_VERSION_KEY)? else { - return Ok(0); - }; - let invalid = || storage_error(py, "SQLite history schema metadata is invalid"); - match optional_text(&value)? { - Some(text) => text.trim().parse::().map_err(|_| invalid()), - None => exact_int(&value).ok_or_else(invalid), - } - } - - fn namespace(connection: &Bound<'_, PyAny>) -> PyResult { - let py: Python<'_> = connection.py(); - match Self::metadata_value(connection, SQLITE_NAMESPACE_KEY)? { - Some(value) => Ok(value.str()?.extract()?), - None => Err(storage_error( - py, - "SQLite history cursor namespace is missing", - )), - } - } - - fn stored_event<'py>(row: &Bound<'py, PyAny>, namespace: &str) -> PyResult> { - let py: Python<'py> = row.py(); - let storage_id: i64 = column(row, "storage_id")?.extract()?; - stored_event( - py, - ( - storage_id, - event_cursor(Dialect::Sqlite, namespace, storage_id), - ), - parsed_timestamp(&column(row, "received_at")?)?, - &column(row, "payload_json")?, - ) - } - - fn read_all_runs(connection: &Bound<'_, PyAny>) -> PyResult> { - execute_plain(connection, SQLITE_ALL_RUNS)? - .try_iter()? - .map(|row| row.and_then(|row| stored_run(&row))) - .collect() - } - - fn replace_projection(connection: &Bound<'_, PyAny>, projected: &[RunRow]) -> PyResult<()> { - let py: Python<'_> = connection.py(); - execute_plain(connection, SQLITE_DELETE_RUNS)?; - if let Ok(connection) = connection.cast::() { - for run in projected { - connection.get().execute_native( - py, - SQLITE_RUN_INSERT, - &sqlite_params(&run_insert_query(Dialect::Sqlite, run).params)?, - )?; - } - return Ok(()); - } - let rows: Vec> = projected - .iter() - .map(|run| { - python_params(py, &run_insert_query(Dialect::Sqlite, run).params) - .map(Bound::into_any) - }) - .collect::>()?; - connection.call_method1("executemany", (SQLITE_RUN_INSERT, tuple_of(py, rows)?))?; - Ok(()) - } - - fn append_prepared<'py>( - connection: &Bound<'py, PyAny>, - prepared: &[PreparedEvent], - ) -> PyResult>> { - let py: Python<'py> = connection.py(); - let mut stored: Vec> = Vec::with_capacity(prepared.len()); - let mut namespace: Option = None; - for event in prepared { - let existing = fetch_one(&execute( - connection, - SQLITE_EVENT_BY_ID, - &[SqlParam::Text(event.event_id.clone())], - )?)?; - let row: Bound<'py, PyAny> = match existing { - Some(existing) => { - let digest: Option = - optional_text(&column(&existing, "content_digest")?)?; - let payload: Option = - optional_text(&column(&existing, "payload_json")?)?; - checked( - py, - validate_stored_content(event, (digest.as_deref(), payload.as_deref())), - )?; - existing - } - None => { - let query = event_insert_query(Dialect::Sqlite, event, &now(py)?); - let cursor: Bound<'py, PyAny> = execute_query(connection, &query)?; - let storage_id: i64 = match cursor.cast::() { - Ok(cursor) => cursor.get().last_row_id, - Err(_) => native_field(&cursor, "lastrowid")?.extract()?, - }; - fetch_one(&execute( - connection, - SQLITE_EVENT_BY_STORAGE_ID, - &[SqlParam::Integer(storage_id)], - )?)? - .unwrap_or_else(|| py.None().into_bound(py)) - } - }; - if namespace.is_none() { - namespace = Some(Self::namespace(connection)?); - } - let namespace: &str = namespace.as_deref().unwrap_or_default(); - stored.push(Self::stored_event(&row, namespace)?); - } - Ok(stored) - } - - fn append_events_in<'py>( - connection: &Bound<'py, PyAny>, - events: &Bound<'py, PyAny>, - ) -> PyResult>> { - let prepared: Vec = prepared_batch(events)?; - Self::append_prepared(connection, &prepared) - } - - fn migrate_in(&self, slf: &Bound<'_, Self>) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let connection: Bound<'_, PyAny> = native_field(slf, "_connection")?; - let current: i64 = Self::metadata_version(&connection)?; - if !checked(py, schema_migration_needed(Dialect::Sqlite, current))? { - return Ok(()); - } - let namespace: String = new_namespace(py)?; - let migrated = self.transaction(slf, |connection| { - for statement in SQLITE_SCHEMA { - execute_plain(connection, statement)?; - } - execute( - connection, - SQLITE_INSERT_NAMESPACE, - &[ - SqlParam::Text(SQLITE_NAMESPACE_KEY.to_owned()), - SqlParam::Text(namespace.clone()), - ], - )?; - execute( - connection, - SQLITE_INSERT_SCHEMA_VERSION, - &[ - SqlParam::Text(SQLITE_SCHEMA_VERSION_KEY.to_owned()), - SqlParam::Text(SCHEMA_VERSION.to_string()), - ], - )?; - Ok(()) - }); - wrapped(py, migrated, "SQLite history schema migration failed") - } - - fn run_cursor_key( - connection: &Bound<'_, PyAny>, - cursor: &Bound<'_, PyAny>, - ) -> PyResult> { - let py: Python<'_> = connection.py(); - if cursor.is_none() { - return Ok(None); - } - let text: String = optional_text(cursor)?.unwrap_or_default(); - let namespace: String = Self::namespace(connection)?; - let decoded = checked(py, decoded_run_cursor(Dialect::Sqlite, &text, &namespace))?; - let invalid_timestamp = - || rejection(py, HistoryErrorKind::InvalidCursor, RUN_CURSOR_TIMESTAMP); - let created_at = match parsed_iso(py, &decoded.created_at_text)? { - Some(parsed) if !parsed.call_method0("utcoffset")?.is_none() => utc_time(&parsed)?, - _ => return Err(invalid_timestamp()), - }; - let exists = fetch_one(&execute( - connection, - SQLITE_RUN_EXISTS, - &[ - SqlParam::Text(decoded.created_at_text.clone()), - SqlParam::Text(decoded.run_id.clone()), - ], - )?)?; - if exists.is_none() { - return Err(rejection( - py, - HistoryErrorKind::InvalidCursor, - UNKNOWN_RUN_CURSOR, - )); - } - Ok(Some(RunCursorKey { - created_at, - created_at_text: decoded.created_at_text, - run_id: decoded.run_id, - })) - } -} - -#[pymethods] -impl SqliteExecutionHistory { - #[new] - #[pyo3(signature = (*_args, **_kwargs))] - fn new(_args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>) -> Self { - Self { - path: OnceLock::new(), - lock: ReentrantLock::default(), - closed: AtomicBool::new(true), - project_calls: AtomicI64::new(0), - } - } - - #[pyo3(signature = (project_dir = None, *, path = None, busy_timeout_ms = None))] - fn __init__( - slf: &Bound<'_, Self>, - project_dir: Option<&Bound<'_, PyAny>>, - path: Option<&Bound<'_, PyAny>>, - busy_timeout_ms: Option<&Bound<'_, PyAny>>, - ) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let this: &Self = slf.get(); - let fspath = |value: Option<&Bound<'_, PyAny>>| -> PyResult> { - value - .filter(|value| !value.is_none()) - .map(|value| imported(py, "os", "fspath")?.call1((value,))?.extract()) - .transpose() - }; - let project_dir: Option = fspath(project_dir)?; - let path: Option = fspath(path)?; - let working_directory: String = if project_dir.is_none() && path.is_none() { - std::env::current_dir()?.to_string_lossy().into_owned() - } else { - String::new() - }; - let location = SqliteLocation { - project_dir, - path, - working_directory, - busy_timeout_ms: busy_timeout_ms.map_or(Some(SQLITE_BUSY_TIMEOUT_MS), exact_int), - }; - let database: String = checked(py, sqlite_database_path(&location))?; - let busy_timeout_ms: i64 = location.busy_timeout_ms.unwrap_or(SQLITE_BUSY_TIMEOUT_MS); - let _ = this.path.set(database.clone()); - this.closed.store(false, Ordering::SeqCst); - let mut connection: Option> = None; - let opened: PyResult<()> = (|| { - prepare_history_directory(&database)?; - let opened = Bound::new(py, SqliteConnection::open(py, &database)?)?.into_any(); - connection = Some(opened.clone()); - slf.setattr("_connection", &opened)?; - this.with_lock(py, || { - execute_plain(&opened, &format!("PRAGMA busy_timeout = {busy_timeout_ms}"))?; - execute_plain(&opened, SQLITE_FOREIGN_KEYS)?; - if database != MEMORY_PATH { - execute_plain(&opened, SQLITE_JOURNAL_MODE)?.call_method0("fetchone")?; - } - slf.call_method0("_migrate")?; - Ok(()) - })?; - restrict_history_file(&database)?; - slf.call_method0("reconcile")?; - Ok(()) - })(); - let Err(error) = opened else { - return Ok(()); - }; - let is_storage: bool = history_error_of_kind(py, &error, STORAGE_ERROR)?; - let is_open_failure: bool = - error.is_instance_of::(py) || sqlite_failure(py, &error)?; - if !is_storage && !is_open_failure { - return Err(error); - } - if let Some(connection) = connection { - let _ = connection.call_method0("close"); - } - this.closed.store(true, Ordering::SeqCst); - if is_storage { - return Err(error); - } - Err(storage_error_from( - py, - &format!("cannot open SQLite history at {database}"), - error, - )) - } - - /// The database path, or `None` for an in-memory database. - #[getter] - pub(crate) fn path<'py>(&self, py: Python<'py>) -> PyResult> { - let path: &str = self.database_path(); - if path == MEMORY_PATH { - return Ok(py.None().into_bound(py)); - } - imported(py, PATHLIB_MODULE, "Path")?.call1((path,)) - } - - /// The number of incremental projection calls. - #[getter] - pub(crate) fn project_calls(&self) -> i64 { - self.project_calls.load(Ordering::SeqCst) - } - - fn append_event<'py>( - slf: &Bound<'py, Self>, - event: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - let this: &Self = slf.get(); - this.with_lock(py, || { - this.ensure_open(py)?; - slf.call_method1("append_events", (tuple_of(py, vec![event.clone()])?,))? - .get_item(0) - }) - } - - fn append_events<'py>( - slf: &Bound<'py, Self>, - events: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - let this: &Self = slf.get(); - this.with_lock(py, || { - this.ensure_open(py)?; - let pending: Bound<'py, PyTuple> = - PyTuple::new(py, events.try_iter()?.collect::>>()?)?; - let appended = this.transaction(slf, |connection| { - Self::append_events_in(connection, pending.as_any()) - }); - let stored = Self::without_conflicts(py, appended, "SQLite event append failed")?; - tuple_of(py, stored) - }) - } - - #[pyo3(signature = (*, event_filter, after_cursor = None, limit = None))] - #[pyo3(name = "get_events")] - fn stored_events_page<'py>( - slf: &Bound<'py, Self>, - event_filter: &Bound<'py, PyAny>, - after_cursor: Option<&Bound<'py, PyAny>>, - limit: Option<&Bound<'py, PyAny>>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - let this: &Self = slf.get(); - this.with_lock(py, || { - this.ensure_open(py)?; - let limit: Option = limit.map_or(Some(DEFAULT_PAGE_LIMIT), exact_int); - checked(py, validate_page_limit(limit))?; - let limit: i64 = limit.unwrap_or(DEFAULT_PAGE_LIMIT); - let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; - let after: i64 = Self::event_position(&connection, after_cursor)?; - let spec = event_filter_spec(event_filter)?; - let query = event_page_query(Dialect::Sqlite, after, &spec, limit); - let rows: Vec> = wrapped( - py, - execute_query(&connection, &query) - .and_then(|cursor| cursor.try_iter()?.collect::>>()), - "SQLite event read failed", - )?; - let has_more: bool = rows.len() as i64 > limit; - let mut records: Vec> = Vec::new(); - if !rows.is_empty() { - let namespace: String = Self::namespace(&connection)?; - for row in rows.iter().take(limit as usize) { - records.push(Self::stored_event(row, &namespace)?); - } - } - let next_cursor: Bound<'py, PyAny> = match records.last() { - Some(record) => native_field(record, "cursor")?, - None => py.None().into_bound(py), - }; - sqlbuild_python_base::boundary::main::call_with::call_with( - &host_entry(py, ExecutorHost::EventPage)?, - &[ - ("records", tuple_of(py, records)?.into_any()), - ("next_cursor", next_cursor), - ("has_more", has_more.into_bound_py_any(py)?), - ], - ) - }) - } - - #[pyo3(name = "get_run")] - fn run_record<'py>( - slf: &Bound<'py, Self>, - run_id: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - let this: &Self = slf.get(); - this.with_lock(py, || { - this.ensure_open(py)?; - let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; - let row = - fetch_one(&connection.call_method1("execute", (SQLITE_RUN_BY_ID, (run_id,)))?)?; - match row { - Some(row) => crate::bindings::_helpers::execution_history::conversion::run_record( - py, - &stored_run(&row)?, - ), - None => Ok(py.None().into_bound(py)), - } - }) - } - - #[pyo3(signature = (*, run_filter, after_cursor = None, limit = None))] - #[pyo3(name = "get_runs")] - fn run_page<'py>( - slf: &Bound<'py, Self>, - run_filter: &Bound<'py, PyAny>, - after_cursor: Option<&Bound<'py, PyAny>>, - limit: Option<&Bound<'py, PyAny>>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - let this: &Self = slf.get(); - this.with_lock(py, || { - this.ensure_open(py)?; - let limit: Option = limit.map_or(Some(DEFAULT_PAGE_LIMIT), exact_int); - checked(py, validate_page_limit(limit))?; - let limit: i64 = limit.unwrap_or(DEFAULT_PAGE_LIMIT); - let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; - let none = py.None().into_bound(py); - let after = Self::run_cursor_key(&connection, after_cursor.unwrap_or(&none))?; - let spec = run_filter_spec(run_filter)?; - let query = run_page_query(Dialect::Sqlite, after.as_ref(), &spec, limit); - let rows: Vec> = execute_query(&connection, &query)? - .try_iter()? - .collect::>()?; - let runs: Vec = rows - .iter() - .take(limit as usize) - .map(stored_run) - .collect::>()?; - let next_cursor: Bound<'py, PyAny> = match runs.last() { - Some(run) => { - let namespace: String = Self::namespace(&connection)?; - checked(py, run_cursor(Dialect::Sqlite, &namespace, run))? - .into_bound_py_any(py)? - } - None => py.None().into_bound(py), - }; - sqlbuild_python_base::boundary::main::call_with::call_with( - &host_entry(py, ExecutorHost::RunPage)?, - &[ - ("records", run_records(py, &runs)?.into_any()), - ("next_cursor", next_cursor), - ( - "has_more", - (rows.len() as i64 > limit).into_bound_py_any(py)?, - ), - ], - ) - }) - } - - fn project<'py>( - slf: &Bound<'py, Self>, - stored_events: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - let this: &Self = slf.get(); - this.with_lock(py, || { - this.ensure_open(py)?; - this.project_calls.fetch_add(1, Ordering::SeqCst); - let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; - let current: Vec = Self::read_all_runs(&connection)?; - let facts = projection_facts(stored_events)?; - let projected: Vec = checked(py, project_runs(facts, current))?; - let records: Bound<'py, PyTuple> = run_records(py, &projected)?; - slf.call_method1("_publish_projection", (&records,))?; - Ok(records) - }) - } - - fn rebuild_from_events<'py>( - slf: &Bound<'py, Self>, - stored_events: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - let this: &Self = slf.get(); - this.with_lock(py, || { - this.ensure_open(py)?; - let facts = projection_facts(stored_events)?; - let projected: Vec = checked(py, project_runs(facts, Vec::new()))?; - let records: Bound<'py, PyTuple> = run_records(py, &projected)?; - slf.call_method1("_publish_projection", (&records,))?; - Ok(records) - }) - } - - /// Atomically append immutable facts and publish their run projection. - fn append_and_project<'py>( - slf: &Bound<'py, Self>, - events: &Bound<'py, PyAny>, - ) -> PyResult> { - let py: Python<'py> = slf.py(); - let this: &Self = slf.get(); - this.with_lock(py, || { - this.ensure_open(py)?; - let pending: Bound<'py, PyTuple> = - PyTuple::new(py, events.try_iter()?.collect::>>()?)?; - let appended = this.transaction(slf, |connection| { - let stored: Vec> = - Self::append_events_in(connection, pending.as_any())?; - let stored_tuple: Bound<'py, PyTuple> = tuple_of(py, stored.clone())?; - let facts = projection_facts(stored_tuple.as_any())?; - let projected = checked(py, project_runs(facts, Self::read_all_runs(connection)?))?; - Self::replace_projection(connection, &projected)?; - Ok(stored) - }); - let stored = Self::without_conflicts( - py, - appended, - "SQLite atomic append and projection failed", - )?; - tuple_of(py, stored) - }) - } - - /// Rebuild the disposable run projection from all durable event facts. - fn reconcile<'py>(slf: &Bound<'py, Self>) -> PyResult> { - let py: Python<'py> = slf.py(); - let this: &Self = slf.get(); - this.with_lock(py, || { - this.ensure_open(py)?; - let reconciled = this.transaction(slf, |connection| { - let records: Bound<'py, PyAny> = slf.call_method0("_read_all_events")?; - let facts = projection_facts(&records)?; - let projected: Vec = checked(py, project_runs(facts, Vec::new()))?; - Self::replace_projection(connection, &projected)?; - run_records(py, &projected) - }); - wrapped(py, reconciled, "SQLite reconciliation failed") - }) - } - - /// Verify connectivity, integrity, and the supported schema revision. - fn check_health(slf: &Bound<'_, Self>) -> PyResult { - let py: Python<'_> = slf.py(); - let this: &Self = slf.get(); - this.with_lock(py, || { - this.ensure_open(py)?; - let connection: Bound<'_, PyAny> = native_field(slf, "_connection")?; - let Some(result) = fetch_one(&execute_plain(&connection, SQLITE_QUICK_CHECK)?)? else { - return Ok(false); - }; - if optional_text(&result.get_item(0)?)?.as_deref() != Some(SQLITE_HEALTHY_CHECK_RESULT) - { - return Ok(false); - } - let version: i64 = slf.call_method0("get_schema_version")?.extract()?; - Ok(version == SCHEMA_VERSION) - }) - } - - #[pyo3(name = "get_schema_version")] - fn schema_version(slf: &Bound<'_, Self>) -> PyResult { - let py: Python<'_> = slf.py(); - let this: &Self = slf.get(); - this.with_lock(py, || { - this.ensure_open(py)?; - Self::metadata_version(&native_field(slf, "_connection")?) - }) - } - - #[pyo3(signature = (*, target_version = None))] - fn upgrade_schema(slf: &Bound<'_, Self>, target_version: Option) -> PyResult { - let py: Python<'_> = slf.py(); - let this: &Self = slf.get(); - this.with_lock(py, || { - this.ensure_open(py)?; - let target: i64 = target_version.unwrap_or(SCHEMA_VERSION); - checked(py, validate_schema_target(Dialect::Sqlite, target))?; - slf.call_method0("_migrate")?; - Ok(target) - }) - } - - fn close(slf: &Bound<'_, Self>) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let this: &Self = slf.get(); - this.with_lock(py, || { - if this.closed.load(Ordering::SeqCst) { - return Ok(()); - } - let closed = native_field(slf, "_connection") - .and_then(|connection| connection.call_method0("close")); - this.closed.store(true, Ordering::SeqCst); - closed.map(|_| ()) - }) - } - - fn dispose(slf: &Bound<'_, Self>) -> PyResult<()> { - slf.call_method0("close")?; - Ok(()) - } - - fn __enter__<'py>(slf: &Bound<'py, Self>) -> PyResult> { - let py: Python<'py> = slf.py(); - let this: &Self = slf.get(); - this.with_lock(py, || this.ensure_open(py))?; - Ok(slf.clone()) - } - - #[pyo3(signature = (*_exc_info))] - fn __exit__(slf: &Bound<'_, Self>, _exc_info: &Bound<'_, PyTuple>) -> PyResult> { - slf.call_method0("close")?; - Ok(None) - } - - fn _migrate(slf: &Bound<'_, Self>) -> PyResult<()> { - slf.get().migrate_in(slf) - } - - fn _read_all_events<'py>(slf: &Bound<'py, Self>) -> PyResult> { - let py: Python<'py> = slf.py(); - let connection: Bound<'py, PyAny> = native_field(slf, "_connection")?; - let rows: Vec> = execute_plain(&connection, SQLITE_ALL_EVENTS)? - .try_iter()? - .collect::>()?; - let mut records: Vec> = Vec::with_capacity(rows.len()); - if !rows.is_empty() { - let namespace: String = Self::namespace(&connection)?; - for row in &rows { - records.push(Self::stored_event(row, &namespace)?); - } - } - tuple_of(py, records) - } - - fn _publish_projection(slf: &Bound<'_, Self>, projected: &Bound<'_, PyAny>) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let runs: Vec = run_rows(projected)?; - let published = slf.get().transaction(slf, |connection| { - Self::replace_projection(connection, &runs) - }); - wrapped(py, published, "SQLite run projection publication failed") - } -} - -impl SqliteExecutionHistory { - /// `result` with integrity conflicts kept and `sqlite3.Error` wrapped as `message`. - fn without_conflicts(py: Python<'_>, result: PyResult, message: &str) -> PyResult { - match result { - Err(error) if history_error_of_kind(py, &error, INTEGRITY_CONFLICT)? => Err(error), - result => wrapped(py, result, message), - } - } - - fn event_position( - connection: &Bound<'_, PyAny>, - cursor: Option<&Bound<'_, PyAny>>, - ) -> PyResult { - let py: Python<'_> = connection.py(); - let Some(cursor) = cursor.filter(|cursor| !cursor.is_none()) else { - return Ok(0); - }; - let text: String = optional_text(cursor)?.unwrap_or_default(); - let namespace: String = Self::namespace(connection)?; - let position: i64 = checked( - py, - event_cursor_position(Dialect::Sqlite, Some(&text), &namespace), - )?; - let exists = fetch_one(&execute( - connection, - SQLITE_EVENT_EXISTS, - &[SqlParam::Integer(position)], - )?)?; - if exists.is_none() { - return Err(rejection( - py, - HistoryErrorKind::InvalidCursor, - UNKNOWN_EVENT_CURSOR, - )); - } - Ok(position) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index d1d5094f01..4c5e9c4568 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -2,7 +2,6 @@ pub(crate) mod orchestration_sensors; pub(crate) mod registration; -pub(crate) mod resource_costs_state; pub(crate) mod setup_context; pub(crate) mod statement_report; pub(crate) mod warehouse_metadata; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index 412cfbcf3b..78ee0b8dc0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -69,8 +69,7 @@ use crate::bindings::_helpers::record_behaviours::values::{ finish_fingerprint_set, logical_type_display_name, sql_value_kind, }; use crate::bindings::_helpers::{ - build_progress, compute_logs, dbt_events, dbt_interop, event_exporting, execution_history, - result_output, + build_progress, compute_logs, dbt_events, dbt_interop, event_exporting, result_output, }; use sqlbuild_python_base::records::models::{GetterEntry, RecordBehaviours}; use sqlbuild_python_base::records::types::ClassMethod; @@ -368,7 +367,7 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ sqlbuild_python_base::diagnostics::constants::GETTERS, sqlbuild_python_front::discovery::constants::GETTERS, event_exporting::getters::GETTERS, - execution_history::getters::GETTERS, + sqlbuild_python_front::execution_history::constants::GETTERS, sqlbuild_python_project::graph::constants::GETTERS, sqlbuild_python_front::model_config::constants::GETTERS, sqlbuild_python_planner::planner_entries::constants::GETTERS, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs index 57dcaef5a2..a33c3d3e0d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs @@ -21,8 +21,6 @@ use crate::bindings::_helpers::dagster_invocation::access::string_tuple as texts use crate::bindings::_helpers::diff_report::command::target_max_full_rows; use crate::bindings::_helpers::diff_report::facts::diff_facts; use crate::bindings::_helpers::diff_report::support::{error_text, matches_class}; -use crate::bindings::_helpers::execution_history::conversion::nullable_text; -use crate::bindings::_helpers::execution_history::statements::fetch_one; use crate::bindings::_helpers::review_command::planning::ReviewSettings; use crate::bindings::_helpers::review_command::support::review_error; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -32,6 +30,8 @@ use sqlbuild_python_base::build_command::main::text::text; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_compile::compile_driver::main::stages::run_diff_pipeline::run_diff_pipeline; +use sqlbuild_python_front::execution_history::main::fetch_one::fetch_one; +use sqlbuild_python_front::execution_history::main::nullable_text::nullable_text; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; const KEYED_CHECK: &str = "keyed comparison"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs index e35bbe3e74..1387fecf1a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/planning.rs @@ -22,7 +22,6 @@ use sqlbuild_config::project::models::ReviewConfig; use crate::bindings::_helpers::change_selection::base_snapshot::{base_comparison, base_variant}; use crate::bindings::_helpers::change_selection::definition_snapshot::project_wide_keys; use crate::bindings::_helpers::change_selection::selection::{ChangedProject, changed_project}; -use crate::bindings::_helpers::execution_history::conversion::nullable_text; use crate::bindings::_helpers::review_command::support::review_error; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::build_command::main::boolean::boolean; @@ -36,6 +35,7 @@ use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::reso use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_front::discovery::main::project::configured_git_base_ref::configured_git_base_ref; use sqlbuild_python_front::discovery::main::project::configured_review::configured_review; +use sqlbuild_python_front::execution_history::main::nullable_text::nullable_text; use sqlbuild_python_project::compiled_views::main::compiled_models::dep_rows::dep_rows; use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/support.rs index d615eec00c..c37c30d1fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/support.rs @@ -11,7 +11,6 @@ use sqlbuild_analysis::review_command::models::ReviewFailure; use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::clone::clone_progress::write_line; -use crate::bindings::_helpers::execution_history::conversion::nullable_text; use crate::bindings::_helpers::janitor::janitor_keys::{now_utc, timestamp_fields}; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -19,6 +18,7 @@ use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_front::execution_history::main::nullable_text::nullable_text; /// The code of an unexpected failure inside a review. const UNEXPECTED_CODE: &str = "C760"; diff --git a/fensu.toml b/fensu.toml index 5b730b140a..1cf02410e9 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1439,11 +1439,11 @@ paths = ["crates/sqlbuild-python-compile/src/analysis/_helpers/queries.rs", "cra reason = "Each bindings module registers its own functions and classes with the native module." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs"] +paths = ["crates/sqlbuild-python-front/src/execution_history/_helpers/postgres.rs", "crates/sqlbuild-python-front/src/execution_history/_helpers/sqlite.rs"] reason = "PostgreSQL and SQLite execution history each read a page through their own connection, locking and cursor keys; filters and record conversion come from conversion.rs." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/prepared_artifacts.rs", "crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs", "crates/sqlbuild-python-base/src/diagnostics/_helpers/log_records.rs", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/fix/fix_session.rs", "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs"] +paths = ["crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/prepared_artifacts.rs", "crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs", "crates/sqlbuild-python-base/src/diagnostics/_helpers/log_records.rs", "crates/sqlbuild-python-front/src/execution_history/_helpers/conversion.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/fix/fix_session.rs", "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs"] reason = "Each turns the one Python exception its own call can expect into None; the calls and the exceptions they accept differ." [[dupes.allowlist]] @@ -1675,7 +1675,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 66768 } +thresholds = { max_crate_code_lines = 63970 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] @@ -2070,7 +2070,7 @@ paths = ["src/sqlbuild/adapter/contract/classes/*", "src/sqlbuild/adapters/*/cla reason = "tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py requires every first-class adapter to define each StrictAdapter method itself instead of inheriting it from BaseAdapter." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs", "crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs", "crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs", "crates/sqlbuild-python/src/bindings/_helpers/native_host/resource_costs_state.rs"] +paths = ["crates/sqlbuild-python-front/src/native_host/_helpers/audit_results_state.rs", "crates/sqlbuild-python-front/src/native_host/_helpers/consumption_state.rs", "crates/sqlbuild-python-front/src/native_host/_helpers/observations_state.rs", "crates/sqlbuild-python-front/src/resource_costs/_helpers/resource_costs_state.rs"] reason = "Each state table has its own Python-facing create and prune bindings. The DDL and table naming live in the shared fixed_state_table_sql and state_table_name; each prune keeps its own retention rule." [[dupes.allowlist]] @@ -2124,6 +2124,7 @@ edge_paths = [ "crates/sqlbuild-python-base/src/records/main/registry/native_attr.rs", "crates/sqlbuild-python-base/src/warehouse_access", "crates/sqlbuild-python-front/src/native_host", + "crates/sqlbuild-python-front/src/resource_costs", "crates/sqlbuild-python-project/src/native_host", "crates/sqlbuild-python/src/bindings/_helpers/native_host", ] @@ -2378,6 +2379,15 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/discovery/main/records/discover_schema_files.rs=1", "crates/sqlbuild-python-front/src/discovery/main/records/discover_test_files.rs=2", "crates/sqlbuild-python-front/src/discovery/main/snapshots/escaped.rs=1", + "crates/sqlbuild-python-front/src/execution_history/_helpers/conversion.rs=15", + "crates/sqlbuild-python-front/src/execution_history/_helpers/functions.rs=2", + "crates/sqlbuild-python-front/src/execution_history/_helpers/postgres.rs=2", + "crates/sqlbuild-python-front/src/execution_history/_helpers/sqlite.rs=6", + "crates/sqlbuild-python-front/src/execution_history/_helpers/statements.rs=3", + "crates/sqlbuild-python-front/src/execution_history/main/fetch_one.rs=1", + "crates/sqlbuild-python-front/src/execution_history/main/parsed_iso.rs=2", + "crates/sqlbuild-python-front/src/execution_history/main/utc.rs=1", + "crates/sqlbuild-python-front/src/execution_history/models.rs=46", "crates/sqlbuild-python-front/src/macro_bridge/_helpers/module_digests.rs=2", "crates/sqlbuild-python-front/src/macro_bridge/_helpers/store_keys.rs=23", "crates/sqlbuild-python-front/src/macro_bridge/main/context_store_token.rs=2", @@ -2724,11 +2734,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs=18", - "crates/sqlbuild-python/src/bindings/_helpers/execution_history/functions.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs=27", - "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs=27", - "crates/sqlbuild-python/src/bindings/_helpers/execution_history/statements.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs=1", @@ -2811,6 +2816,7 @@ edge_paths = [ "crates/sqlbuild-python-base/src/records/_helpers/native_classes/registry.rs", "crates/sqlbuild-python-base/src/warehouse_access", "crates/sqlbuild-python-front/src/native_host", + "crates/sqlbuild-python-front/src/resource_costs", "crates/sqlbuild-python-project/src/native_host", "crates/sqlbuild-python/src/bindings/_helpers/native_host", ] @@ -3104,7 +3110,7 @@ module_layers = [ "diff_report=commands", "discovery=registry", "event_exporting=output", - "execution_history=front", + "execution_history=registry", "exported_constants=registry", "fingerprints=registry", "format_command=commands", @@ -3128,7 +3134,6 @@ module_layers = [ "native_host=front", "native_host/orchestration_sensors=commands", "native_host/setup_context=commands", - "native_host/resource_costs_state=front", "native_host/warehouse_metadata=commands", "observations=plan", "output_capture=output", @@ -3849,6 +3854,42 @@ reason = "Each is one rule family's entry point: return early when no rule is se paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs"] reason = "Each builds its own dbt manifest record type with that type's fields; only the record-building shape is alike." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/cursor/_helpers/scalars.rs", "crates/sqlbuild-python-front/src/execution_history/main/parsed_iso.rs", "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs"] +reason = "Each turns the one Python exception its own call can expect into None; the calls and the exceptions they accept differ." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/execution_history/models.rs"] +reason = "Each PostgreSQL history method runs its own statement over the events it prepares; only the open-check and event preparation are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/execution_history/models.rs"] +reason = "PostgreSQL and SQLite execution history each read a page through their own connection, locking and cursor keys; filters and record conversion come from conversion.rs." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/execution_history/main/register_functions.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/execution_history/models.rs"] +reason = "Projection and rebuild run different statements under the same lock; only the lock-and-open shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/execution_history/models.rs"] +reason = "Each forwards to its own Python-side hook with its own arguments; only the keyword call is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/execution_history/models.rs"] +reason = "PostgreSQL and SQLite execution history each do this through their own connection, locking and statements; only the open-check shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/execution_history/models.rs"] +reason = "PostgreSQL and SQLite execution history each do this through their own connection, locking and statements; only the open-check shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-front/src/execution_history/models.rs"] +reason = "Each backend's constructor initialises its own state fields; only the closed flag and call counter are alike." + [[dupes.allowlist]] paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs"] reason = "Each is its own Python entry point with its own keyword contract and dbt command shape; both end in the shared argv builder." From d2e7b89443e66a5689bcad25e4d807a5ba0bc967 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 10:41:42 +0000 Subject: [PATCH 2/2] refactor: move event exporting into sqlbuild-python-output The native lifecycle-event export dispatcher, its workers, export policy and attribute getters move from the Python bindings crate into sqlbuild-python-output. The bindings crate keeps only the registration, which now imports the classes and the interpreter-exit hook from there. --- .../event_exporting/_helpers}/dispatcher.rs | 486 ++---------- .../src/event_exporting/_helpers}/getters.rs | 50 +- .../src/event_exporting/_helpers/mod.rs | 4 + .../src/event_exporting/_helpers}/policy.rs | 16 +- .../src/event_exporting/_helpers}/workers.rs | 275 +------ .../src/event_exporting/constants.rs | 50 ++ .../src/event_exporting/main/mod.rs | 2 + .../event_exporting/main/register_policy.rs | 13 + .../stop_event_exporter_threads_function.rs | 10 + .../src/event_exporting/mod.rs | 6 + .../src/event_exporting/models.rs | 691 ++++++++++++++++++ crates/sqlbuild-python-output/src/lib.rs | 1 + .../bindings/_helpers/cli/observability.rs | 2 +- .../bindings/_helpers/event_exporting/mod.rs | 4 - .../_helpers/event_exporting/registration.rs | 15 +- .../_helpers/record_behaviours/constants.rs | 4 +- fensu.toml | 34 +- 17 files changed, 898 insertions(+), 765 deletions(-) rename crates/{sqlbuild-python/src/bindings/_helpers/event_exporting => sqlbuild-python-output/src/event_exporting/_helpers}/dispatcher.rs (55%) rename crates/{sqlbuild-python/src/bindings/_helpers/event_exporting => sqlbuild-python-output/src/event_exporting/_helpers}/getters.rs (80%) create mode 100644 crates/sqlbuild-python-output/src/event_exporting/_helpers/mod.rs rename crates/{sqlbuild-python/src/bindings/_helpers/event_exporting => sqlbuild-python-output/src/event_exporting/_helpers}/policy.rs (77%) rename crates/{sqlbuild-python/src/bindings/_helpers/event_exporting => sqlbuild-python-output/src/event_exporting/_helpers}/workers.rs (62%) create mode 100644 crates/sqlbuild-python-output/src/event_exporting/constants.rs create mode 100644 crates/sqlbuild-python-output/src/event_exporting/main/mod.rs create mode 100644 crates/sqlbuild-python-output/src/event_exporting/main/register_policy.rs create mode 100644 crates/sqlbuild-python-output/src/event_exporting/main/stop_event_exporter_threads_function.rs create mode 100644 crates/sqlbuild-python-output/src/event_exporting/mod.rs create mode 100644 crates/sqlbuild-python-output/src/event_exporting/models.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs b/crates/sqlbuild-python-output/src/event_exporting/_helpers/dispatcher.rs similarity index 55% rename from crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs rename to crates/sqlbuild-python-output/src/event_exporting/_helpers/dispatcher.rs index 39a80d892f..7d1020f38c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs +++ b/crates/sqlbuild-python-output/src/event_exporting/_helpers/dispatcher.rs @@ -5,42 +5,31 @@ use std::sync::{Arc, Mutex, MutexGuard}; use std::time::{Duration, Instant}; use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyResult, Python, pyclass, pyfunction, pymethods, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyString}; -use sqlbuild_core::command_output::constants::COMMAND_OUTPUT_RECORD_TYPE; use sqlbuild_core::event_export::constants::{ DEFAULT_EVENT_EXPORT_HEALTH_INTERVAL_SECONDS, DEFAULT_EVENT_EXPORT_INVOCATION_TIMEOUT_SECONDS, DEFAULT_EVENT_EXPORT_NOTIFICATION_QUEUE_CAPACITY, DEFAULT_EVENT_EXPORT_QUEUE_CAPACITY, - DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, INVOCATION_START_EVENT_TYPE, - INVOCATION_TERMINAL_EVENT_TYPES, OUTCOME_EVENT_TYPES, OUTCOME_RESERVE_FRACTION, + DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, }; -use sqlbuild_core::event_export::main::drop_lifecycle_attempts::drop_lifecycle_attempts; use sqlbuild_core::event_export::main::export_snapshot::export_snapshot; -use sqlbuild_core::event_export::main::queue_lifecycle_event::queue_lifecycle_event; -use sqlbuild_core::event_export::main::route_lifecycle_event::route_lifecycle_event; use sqlbuild_core::event_export::models::{ - EventQueue, ExportPolicy, ExportSeverity, ExportSnapshot, ExporterCounts, ExporterFilter, - QueuedEvent, SnapshotInputs, + EventQueue, ExportSeverity, ExportSnapshot, ExporterCounts, ExporterFilter, SnapshotInputs, }; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::event_exporting::workers::{DispatcherWorker, Lane, WorkerRole}; +use crate::event_exporting::_helpers::workers::Lane; +use crate::event_exporting::_helpers::workers::WorkerRole; +use crate::event_exporting::models::DispatcherWorker; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::event_exporting::constants::DISPATCHER_THREAD_NAME; use sqlbuild_python_base::event_exporting::constants::EXIT_JOIN_SECONDS; -use sqlbuild_python_base::event_exporting::constants::HEALTH_THREAD_NAME; use sqlbuild_python_base::event_exporting::constants::IDLE_WAIT_SECONDS; -use sqlbuild_python_base::event_exporting::constants::JOIN_GRACE_SECONDS; use sqlbuild_python_base::event_exporting::constants::NO_SECONDS; -use sqlbuild_python_base::event_exporting::constants::NOTIFIER_THREAD_NAME; use sqlbuild_python_base::event_exporting::constants::OUTPUT_SINK_THREAD_NAME; use sqlbuild_python_base::event_exporting::constants::POLL_SECONDS; use sqlbuild_python_base::event_exporting::constants::WORKER_ATTRIBUTE; -use sqlbuild_python_base::event_exporting::main::export_policy::export_policy; use sqlbuild_python_base::event_exporting::main::input_error::input_error; use sqlbuild_python_base::event_exporting::main::severity_from::severity_from; use sqlbuild_python_base::event_exporting::main::state_error::state_error; @@ -54,6 +43,7 @@ use sqlbuild_python_base::native_threads::main::signal_interpreter_exit::signal_ use sqlbuild_python_base::records::main::fields::field::field as native_field; /// One bound lifecycle-event sink. +#[derive(Debug)] pub(crate) struct Exporter { pub(crate) name: String, pub(crate) function: Py, @@ -61,13 +51,15 @@ pub(crate) struct Exporter { } /// One bound command-output sink and the streams it accepts. +#[derive(Debug)] pub(crate) struct OutputSink { - function: Py, - provider_arguments: Py, - streams: Vec, + pub(crate) function: Py, + pub(crate) provider_arguments: Py, + pub(crate) streams: Vec, } /// A queued event with the policy dimensions its failures report. +#[derive(Debug)] pub(crate) struct Queued { pub(crate) event: Py, pub(crate) kind: String, @@ -75,17 +67,18 @@ pub(crate) struct Queued { } /// Delivery state, read and written only while holding the dispatcher's Python condition. +#[derive(Debug)] pub(crate) struct State { pub(crate) bound: bool, pub(crate) exporters: Vec>, pub(crate) filters: Vec, pub(crate) counts: Vec, - output_sinks: Arc>, + pub(crate) output_sinks: Arc>, /// Calls queued to the single command-output sink thread, once it has started. - output_lane: Option>, + pub(crate) output_lane: Option>, pub(crate) queue: EventQueue, - sequence: u64, - accepting: bool, + pub(crate) sequence: u64, + pub(crate) accepting: bool, pub(crate) deadline: Option, /// When only outcome events remain deliverable, ahead of the deadline. pub(crate) reserve_at: Option, @@ -112,10 +105,10 @@ pub(crate) struct State { pub(crate) last_charged: Option, pub(crate) live_invocations: HashSet, pub(crate) next_invocation: u64, - dispatcher_running: bool, - shutdown_started: bool, - shutdown_complete: bool, - final_summary: Option>, + pub(crate) dispatcher_running: bool, + pub(crate) shutdown_started: bool, + pub(crate) shutdown_complete: bool, + pub(crate) final_summary: Option>, pub(crate) failures: VecDeque, pub(crate) notification_stopping: bool, pub(crate) health_stopping: bool, @@ -127,31 +120,21 @@ pub(crate) struct State { } /// What the dispatcher and its threads share. +#[derive(Debug)] pub(crate) struct Shared { - condition: Py, + pub(crate) condition: Py, /// The `SimpleQueue` every delivery thread reports `(token, error)` outcomes to. pub(crate) results: Py, /// Attempt tokens a delivery thread claimed (`True`) before running, or the dispatcher cancelled (`False`). pub(crate) claims: Py, - state: Mutex, + pub(crate) state: Mutex, pub(crate) invocation_timeout: Duration, - shutdown_timeout: Duration, + pub(crate) shutdown_timeout: Duration, pub(crate) health_interval: Duration, - notification_capacity: usize, + pub(crate) notification_capacity: usize, pub(crate) failure_callback: Option>, pub(crate) summary_callback: Option>, - notifications: bool, -} - -/// Deliver filtered lifecycle events off execution threads with bounded memory. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct EventExporterDispatcher { - shared: Arc, - /// Event kinds this command never exports, such as compile's phase operations. - withheld_kinds: Vec, - thread: Py, - notification_thread: Option>, - health_thread: Option>, + pub(crate) notifications: bool, } /// The dispatcher's Python condition, entered while alive and exited on drop. @@ -166,12 +149,12 @@ pub(crate) enum Delivery { } /// Validated dispatcher limits, in seconds and items. -struct Limits { - queue_capacity: usize, - shutdown_timeout: Duration, - invocation_timeout: Duration, - notification_capacity: usize, - health_interval: Duration, +pub(crate) struct Limits { + pub(crate) queue_capacity: usize, + pub(crate) shutdown_timeout: Duration, + pub(crate) invocation_timeout: Duration, + pub(crate) notification_capacity: usize, + pub(crate) health_interval: Duration, } impl Drop for Held<'_> { @@ -188,14 +171,14 @@ fn seconds(value: f64) -> Duration { Duration::try_from_secs_f64(value.max(NO_SECONDS)).unwrap_or(Duration::MAX) } -fn remaining_seconds(deadline: Instant) -> f64 { +pub(crate) fn remaining_seconds(deadline: Instant) -> f64 { deadline .saturating_duration_since(Instant::now()) .as_secs_f64() } /// End a retired command-output sink thread once it has run every call queued before. -fn retire_output_lane(py: Python<'_>, retired: Option>) -> PyResult<()> { +pub(crate) fn retire_output_lane(py: Python<'_>, retired: Option>) -> PyResult<()> { if let Some(calls) = retired { calls.bind(py).call_method1("put", (py.None(),))?; } @@ -203,7 +186,7 @@ fn retire_output_lane(py: Python<'_>, retired: Option>) -> PyResult<() } /// `thread.join(timeout=seconds)` when the thread is alive. -fn join(thread: &Bound<'_, PyAny>, seconds: f64) -> PyResult<()> { +pub(crate) fn join(thread: &Bound<'_, PyAny>, seconds: f64) -> PyResult<()> { if thread.call_method0("is_alive")?.is_truthy()? { call_with( &native_field(thread, "join")?, @@ -214,7 +197,7 @@ fn join(thread: &Bound<'_, PyAny>, seconds: f64) -> PyResult<()> { } /// Bound exporters read from `BoundEventExporter` values or objects with the same attributes. -fn read_exporters( +pub(crate) fn read_exporters( exporters: &Bound<'_, PyAny>, ) -> PyResult<(Vec>, Vec)> { let mut bound: Vec> = Vec::new(); @@ -247,7 +230,7 @@ fn read_exporters( Ok((bound, filters)) } -fn read_output_sinks(sinks: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn read_output_sinks(sinks: &Bound<'_, PyAny>) -> PyResult> { let mut bound: Vec = Vec::new(); for sink in sinks.try_iter()? { let sink: Bound<'_, PyAny> = sink?; @@ -264,7 +247,7 @@ fn read_output_sinks(sinks: &Bound<'_, PyAny>) -> PyResult> { } /// `health_interval_seconds` when it is a positive, finite non-boolean number. -fn health_interval(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn health_interval(value: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = value.py(); if value.is_none() { return Ok(DEFAULT_EVENT_EXPORT_HEALTH_INTERVAL_SECONDS); @@ -288,7 +271,7 @@ fn health_interval(value: &Bound<'_, PyAny>) -> PyResult { } /// The constructor's limits, validated in the Python dispatcher's order. -fn limits(options: &KeywordArguments<'_>) -> PyResult { +pub(crate) fn limits(options: &KeywordArguments<'_>) -> PyResult { let integer = |name: &str, default: i64| -> PyResult { let value = options.value(name)?; if value.is_none() { @@ -425,7 +408,7 @@ impl Shared { .is_some_and(|deadline| Instant::now() >= deadline) } - fn snapshot(state: &State, final_snapshot: bool) -> ExportSnapshot { + pub(crate) fn snapshot(state: &State, final_snapshot: bool) -> ExportSnapshot { let exporters: Vec<(String, ExporterCounts)> = state .exporters .iter() @@ -456,7 +439,7 @@ impl Shared { } /// Forget one finished invocation and wake a caller waiting for delivery to become idle. - fn invocation_finished(&self, py: Python<'_>, invocation: u64) -> PyResult<()> { + pub(crate) fn invocation_finished(&self, py: Python<'_>, invocation: u64) -> PyResult<()> { let retired: Option> = { let _held = self.hold(py)?; let mut state = self.locked()?; @@ -481,12 +464,12 @@ impl Shared { } /// Whether the dispatcher thread and every sink call have finished. - fn idle(state: &State) -> bool { + pub(crate) fn idle(state: &State) -> bool { state.live_invocations.is_empty() && !state.dispatcher_running } /// Wait until delivery and every sink call are idle, the idle wait ends or interpreter exit begins. - fn await_idle(&self, py: Python<'_>) -> PyResult<()> { + pub(crate) fn await_idle(&self, py: Python<'_>) -> PyResult<()> { let deadline: Instant = Instant::now() + self.invocation_timeout.min(seconds(IDLE_WAIT_SECONDS)); let threads: Vec> = { @@ -518,7 +501,7 @@ impl Shared { } /// Register a running invocation, returning its identity. - fn begin_invocation(&self, py: Python<'_>) -> PyResult { + pub(crate) fn begin_invocation(&self, py: Python<'_>) -> PyResult { let _held = self.hold(py)?; let mut state = self.locked()?; state.next_invocation += 1; @@ -528,7 +511,7 @@ impl Shared { } /// Stop acceptance, delivery, health and notifications so every long-lived thread returns. - fn stop_for_exit(&self, py: Python<'_>) -> PyResult<()> { + pub(crate) fn stop_for_exit(&self, py: Python<'_>) -> PyResult<()> { let _held = self.hold(py)?; let mut state = self.locked()?; state.accepting = false; @@ -541,7 +524,7 @@ impl Shared { } /// The queue feeding this dispatcher's single command-output sink thread, started on first use. - fn output_lane<'py>(&self, py: Python<'py>) -> PyResult> { + pub(crate) fn output_lane<'py>(&self, py: Python<'py>) -> PyResult> { let _held = self.hold(py)?; let existing: Option> = self .locked()? @@ -576,7 +559,7 @@ impl Shared { } /// Call one command-output sink on the sink thread and wait until it returns or exit begins. - fn call_output_sink( + pub(crate) fn call_output_sink( &self, py: Python<'_>, sink: &OutputSink, @@ -613,7 +596,7 @@ impl Shared { } /// The final summary once another caller's shutdown completes. - fn await_shutdown(&self, py: Python<'_>) -> PyResult> { + pub(crate) fn await_shutdown(&self, py: Python<'_>) -> PyResult> { let _held = self.hold(py)?; loop { let state = self.locked()?; @@ -633,7 +616,7 @@ impl Shared { } /// A started daemon thread running `worker`, tagged so the exit hook can stop it. -fn start_worker<'py>( +pub(crate) fn start_worker<'py>( py: Python<'py>, shared: &Arc, (role, name): (WorkerRole, &str), @@ -652,372 +635,3 @@ fn start_worker<'py>( thread.setattr(WORKER_ATTRIBUTE, worker)?; Ok(thread) } - -#[pymethods] -impl EventExporterDispatcher { - #[new] - #[pyo3(signature = (**options))] - pub(crate) fn new(py: Python<'_>, options: Option>) -> PyResult { - let options: KeywordArguments<'_> = KeywordArguments::read( - py, - "EventExporterDispatcher", - options, - ( - &[], - &[ - "exporters", - "queue_capacity", - "shutdown_timeout_seconds", - "invocation_timeout_seconds", - "notification_queue_capacity", - "health_interval_seconds", - "failure_callback", - "summary_callback", - "withheld_kinds", - ], - ), - )?; - let withheld: Bound<'_, PyAny> = options.value("withheld_kinds")?; - let withheld_kinds: Vec = if withheld.is_none() { - Vec::new() - } else { - withheld.extract()? - }; - let limits: Limits = limits(&options)?; - let exporters: Bound<'_, PyAny> = options.value("exporters")?; - let (bound_exporters, filters) = if exporters.is_none() { - (Vec::new(), Vec::new()) - } else { - read_exporters(&exporters)? - }; - let callback = |name: &str| -> PyResult>> { - let value: Bound<'_, PyAny> = options.value(name)?; - Ok((!value.is_none()).then(|| value.unbind())) - }; - let failure_callback: Option> = callback("failure_callback")?; - let summary_callback: Option> = callback("summary_callback")?; - let notifications: bool = failure_callback.is_some() || summary_callback.is_some(); - let has_summary: bool = summary_callback.is_some(); - let shared = Arc::new(Shared { - condition: imported(py, "threading", "Condition")?.call0()?.unbind(), - results: imported(py, "queue", "SimpleQueue")?.call0()?.unbind(), - claims: PyDict::new(py).unbind(), - state: Mutex::new(State { - bound: !exporters.is_none(), - counts: vec![ExporterCounts::default(); bound_exporters.len()], - exporters: bound_exporters, - filters, - output_sinks: Arc::new(Vec::new()), - output_lane: None, - queue: EventQueue::new(limits.queue_capacity), - sequence: 0, - accepting: true, - deadline: None, - reserve_at: None, - reserved: false, - lanes: Vec::new(), - tokens: HashMap::new(), - next_token: 0, - retired: Vec::new(), - abandoned_calls: false, - stopping: false, - force_stop: false, - blocked: HashSet::new(), - stalled: HashMap::new(), - held: Vec::new(), - newly_blocked: Vec::new(), - last_charged: None, - live_invocations: HashSet::new(), - next_invocation: 0, - dispatcher_running: true, - shutdown_started: false, - shutdown_complete: false, - final_summary: None, - failures: VecDeque::new(), - notification_stopping: false, - health_stopping: false, - pending_periodic: None, - periodic_inflight: None, - last_published_periodic: [ - 0, - 0, - 0, - 0, - 0, - 0, - u64::try_from(limits.queue_capacity).unwrap_or(u64::MAX), - ], - final_notification: None, - final_delivered: false, - }), - invocation_timeout: limits.invocation_timeout, - shutdown_timeout: limits.shutdown_timeout, - health_interval: limits.health_interval, - notification_capacity: limits.notification_capacity, - failure_callback, - summary_callback, - notifications, - }); - let notification_thread: Option> = notifications - .then(|| start_worker(py, &shared, (WorkerRole::Notify, NOTIFIER_THREAD_NAME))) - .transpose()?; - let health_thread: Option> = has_summary - .then(|| start_worker(py, &shared, (WorkerRole::Health, HEALTH_THREAD_NAME))) - .transpose()?; - for thread in notification_thread.iter().chain(&health_thread) { - thread.call_method0("start")?; - } - let thread = start_worker(py, &shared, (WorkerRole::Dispatch, DISPATCHER_THREAD_NAME))?; - thread.call_method0("start")?; - Ok(Self { - shared, - withheld_kinds, - thread: thread.unbind(), - notification_thread: notification_thread.map(Bound::unbind), - health_thread: health_thread.map(Bound::unbind), - }) - } - - #[getter] - pub(crate) fn thread(&self, py: Python<'_>) -> Py { - self.thread.clone_ref(py) - } - - #[getter] - pub(crate) fn notification_thread(&self, py: Python<'_>) -> Option> { - self.notification_thread - .as_ref() - .map(|thread| thread.clone_ref(py)) - } - - /// Filter and enqueue eligible exporter attempts without waiting. - fn enqueue(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { - let event_type: String = native_field(event, "event_type")?.extract()?; - let policy: ExportPolicy = export_policy(py, &event_type)?; - if self.withheld_kinds.contains(&policy.kind) { - return Ok(()); - } - let invocation_id: Option = match native_field(event, "invocation_id") { - Ok(value) => value.extract()?, - Err(_) => None, - }; - let shared = &self.shared; - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - if !state.bound { - drop(state); - return Err(state_error( - py, - "event exporters must be bound before enqueue", - )); - } - let counts: Vec = std::mem::take(&mut state.counts); - let (counts, eligible) = route_lifecycle_event(&policy, &state.filters, counts); - state.counts = counts; - if eligible.is_empty() { - return Ok(()); - } - if !state.accepting { - state.counts = drop_lifecycle_attempts(std::mem::take(&mut state.counts), &eligible); - return Ok(()); - } - let sequence: u64 = state.sequence; - state.sequence += 1; - let capacity: usize = state.queue.capacity; - let mut outcome = queue_lifecycle_event( - std::mem::replace(&mut state.queue, EventQueue::new(capacity)), - QueuedEvent { - sequence, - item: Queued { - event: event.clone().unbind(), - kind: policy.kind, - severity: policy.severity, - }, - priority: policy.priority, - invocation_terminal: INVOCATION_TERMINAL_EVENT_TYPES.contains(&event_type.as_str()), - outcome: OUTCOME_EVENT_TYPES.contains(&event_type.as_str()), - invocation_id, - invocation_start: event_type == INVOCATION_START_EVENT_TYPE, - eligible_exporters: eligible, - }, - ); - state.queue = std::mem::replace(&mut outcome.queue, EventQueue::new(capacity)); - for lost in outcome.displaced.iter().chain(&outcome.rejected) { - state.counts = drop_lifecycle_attempts( - std::mem::take(&mut state.counts), - &lost.eligible_exporters, - ); - } - drop(state); - shared.notify_all(py)?; - drop(outcome); - Ok(()) - } - - /// Bind validated exporters once before lifecycle publication starts. - fn bind(&self, py: Python<'_>, exporters: &Bound<'_, PyAny>) -> PyResult<()> { - let (exporters, filters) = read_exporters(exporters)?; - let shared = &self.shared; - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - if state.bound { - drop(state); - return Err(state_error(py, "event exporters are already bound")); - } - state.counts = vec![ExporterCounts::default(); exporters.len()]; - state.exporters = exporters; - state.filters = filters; - state.bound = true; - drop(state); - shared.notify_all(py) - } - - /// Bind explicit command-output sinks before capture starts. - fn bind_command_output_sinks(&self, py: Python<'_>, sinks: &Bound<'_, PyAny>) -> PyResult<()> { - let sinks: Arc> = Arc::new(read_output_sinks(sinks)?); - let previous = { - let _held = self.shared.hold(py)?; - std::mem::replace(&mut self.shared.locked()?.output_sinks, sinks) - }; - drop(previous); - Ok(()) - } - - /// Deliver output records through the already-bound destination providers. - fn export_output(&self, py: Python<'_>, records: &Bound<'_, PyAny>) -> PyResult<()> { - let shared = &self.shared; - let invocation: u64 = shared.begin_invocation(py)?; - let sinks: Arc> = { - let _held = shared.hold(py)?; - Arc::clone(&shared.locked()?.output_sinks) - }; - let mut failed: bool = false; - let delivered = (|| -> PyResult { - for record in records.try_iter()? { - let record: Bound<'_, PyAny> = record?; - let is_output: bool = - native_field(&record, "record_type")?.eq(COMMAND_OUTPUT_RECORD_TYPE)?; - let stream: Option = if is_output { - Some(native_field(&record, "stream")?.extract()?) - } else { - None - }; - for sink in sinks.iter() { - if stream - .as_ref() - .is_some_and(|stream| !sink.streams.contains(stream)) - { - continue; - } - match shared.call_output_sink(py, sink, &record)? { - Delivery::Delivered(sink_failed) => failed |= sink_failed, - Delivery::Abandoned => return Ok(Delivery::Abandoned), - } - } - } - Ok(Delivery::Delivered(failed)) - })(); - if let Ok(Delivery::Abandoned) = delivered { - return Err(state_error( - py, - "output export stopped because the interpreter is exiting", - )); - } - shared.invocation_finished(py, invocation)?; - delivered?; - if failed { - return Err(state_error(py, "one or more output export attempts failed")); - } - Ok(()) - } - - /// Stop acceptance and drain until the fixed shutdown deadline. - fn shutdown(&self, py: Python<'_>) -> PyResult> { - let shared = &self.shared; - let (owns, deadline) = { - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - let owns: bool = !state.shutdown_started; - if owns { - state.shutdown_started = true; - state.accepting = false; - let started: Instant = Instant::now(); - state.deadline = Some(started + shared.shutdown_timeout); - state.reserve_at = Some( - started - + shared - .shutdown_timeout - .mul_f64(1.0 - OUTCOME_RESERVE_FRACTION), - ); - state.stopping = true; - state.health_stopping = true; - } - let deadline: Instant = state.deadline.unwrap_or_else(Instant::now); - drop(state); - shared.notify_all(py)?; - (owns, deadline) - }; - if !owns { - return shared.await_shutdown(py); - } - let thread: &Bound<'_, PyAny> = self.thread.bind(py); - call_with( - &native_field(thread, "join")?, - &[( - "timeout", - PyFloat::new(py, remaining_seconds(deadline) + JOIN_GRACE_SECONDS).into_any(), - )], - )?; - if thread.call_method0("is_alive")?.is_truthy()? { - { - let _held = shared.hold(py)?; - shared.locked()?.force_stop = true; - shared.notify_all(py)?; - } - thread.call_method0("join")?; - } - let snapshot: ExportSnapshot = { - let _held = shared.hold(py)?; - Shared::snapshot(&*shared.locked()?, true) - }; - let summary: Py = EventExportSummary::object(py, snapshot)?; - { - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - state.final_summary = Some(summary.clone_ref(py)); - state.shutdown_complete = true; - if shared.summary_callback.is_some() - && state.final_notification.is_none() - && !state.final_delivered - { - state.final_notification = Some(summary.clone_ref(py)); - } - state.notification_stopping = true; - let retired: Option> = if state.live_invocations.is_empty() { - state.output_lane.take() - } else { - None - }; - drop(state); - shared.notify_all(py)?; - retire_output_lane(py, retired)?; - } - if let Some(health) = &self.health_thread { - join(health.bind(py), EXIT_JOIN_SECONDS)?; - } - Ok(summary) - } - - /// Run `finalizer` on the calling thread, never a daemon delivery thread, once delivery is idle or the idle wait ends. - fn finalize_when_idle(&self, py: Python<'_>, finalizer: Py) -> PyResult<()> { - self.shared.await_idle(py)?; - drop(finalizer.bind(py).call0()); - Ok(()) - } - - /// Return a thread-safe immutable point-in-time snapshot. - fn summary(&self, py: Python<'_>) -> PyResult> { - let snapshot: ExportSnapshot = self.shared.current_snapshot(py)?; - EventExportSummary::object(py, snapshot) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/getters.rs b/crates/sqlbuild-python-output/src/event_exporting/_helpers/getters.rs similarity index 80% rename from crates/sqlbuild-python/src/bindings/_helpers/event_exporting/getters.rs rename to crates/sqlbuild-python-output/src/event_exporting/_helpers/getters.rs index ee7ad4da9a..1248fe68ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/getters.rs +++ b/crates/sqlbuild-python-output/src/event_exporting/_helpers/getters.rs @@ -3,49 +3,15 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::event_exporting::dispatcher::EventExporterDispatcher; +use crate::event_exporting::models::EventExporterDispatcher; use sqlbuild_python_base::event_exporting::models::BoundEventExporter; use sqlbuild_python_base::event_exporting::models::EventExportSummary; use sqlbuild_python_base::event_exporting::models::EventExporterAccounting; use sqlbuild_python_base::event_exporting::models::EventExporterCommandScope; use sqlbuild_python_base::event_exporting::models::EventExporterCounts; use sqlbuild_python_base::event_exporting::models::EventExporterFailure; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[ - GetterEntry { - class: class_of::, - read: read_event_exporter_dispatcher, - }, - GetterEntry { - class: class_of::, - read: read_bound_event_exporter, - }, - GetterEntry { - class: class_of::, - read: read_event_export_summary, - }, - GetterEntry { - class: class_of::, - read: read_event_exporter_accounting, - }, - GetterEntry { - class: class_of::, - read: read_event_exporter_counts, - }, - GetterEntry { - class: class_of::, - read: read_event_exporter_failure, - }, - GetterEntry { - class: class_of::, - read: read_event_exporter_command_scope, - }, -]; - -fn read_event_exporter_dispatcher<'py>( +pub(crate) fn read_event_exporter_dispatcher<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -63,7 +29,7 @@ fn read_event_exporter_dispatcher<'py>( }) } -fn read_bound_event_exporter<'py>( +pub(crate) fn read_bound_event_exporter<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -88,7 +54,7 @@ fn read_bound_event_exporter<'py>( }) } -fn read_event_export_summary<'py>( +pub(crate) fn read_event_export_summary<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -116,7 +82,7 @@ fn read_event_export_summary<'py>( }) } -fn read_event_exporter_accounting<'py>( +pub(crate) fn read_event_exporter_accounting<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -134,7 +100,7 @@ fn read_event_exporter_accounting<'py>( }) } -fn read_event_exporter_counts<'py>( +pub(crate) fn read_event_exporter_counts<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -155,7 +121,7 @@ fn read_event_exporter_counts<'py>( }) } -fn read_event_exporter_failure<'py>( +pub(crate) fn read_event_exporter_failure<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -177,7 +143,7 @@ fn read_event_exporter_failure<'py>( }) } -fn read_event_exporter_command_scope<'py>( +pub(crate) fn read_event_exporter_command_scope<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-output/src/event_exporting/_helpers/mod.rs b/crates/sqlbuild-python-output/src/event_exporting/_helpers/mod.rs new file mode 100644 index 0000000000..94c999df39 --- /dev/null +++ b/crates/sqlbuild-python-output/src/event_exporting/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod dispatcher; +pub(crate) mod getters; +pub(crate) mod policy; +pub(crate) mod workers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs b/crates/sqlbuild-python-output/src/event_exporting/_helpers/policy.rs similarity index 77% rename from crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs rename to crates/sqlbuild-python-output/src/event_exporting/_helpers/policy.rs index e5b4153519..17d9066c29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs +++ b/crates/sqlbuild-python-output/src/event_exporting/_helpers/policy.rs @@ -1,8 +1,6 @@ //! Lifecycle export policy, severity comparison and narrowing for Python callers. -use pyo3::prelude::{ - Bound, PyAny, PyModule, PyModuleMethods, PyResult, Python, pyfunction, wrap_pyfunction, -}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_core::event_export::main::severity_at_least::severity_at_least; use sqlbuild_core::event_export::main::stricter_severity::stricter_severity; @@ -16,7 +14,7 @@ use sqlbuild_python_base::event_exporting::main::severity_object::severity_objec /// `{event_type: (kind, severity, priority)}` for every catalogued lifecycle event, in catalogue order. #[pyfunction] -fn lifecycle_export_policy_catalog(py: Python<'_>) -> PyResult> { +pub(crate) fn lifecycle_export_policy_catalog(py: Python<'_>) -> PyResult> { let policies: Bound<'_, PyDict> = PyDict::new(py); for definition in catalog(CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION).unwrap_or_default() { let event_type: &str = definition.event_type; @@ -36,7 +34,7 @@ fn lifecycle_export_policy_catalog(py: Python<'_>) -> PyResult /// Whether a catalogued severity meets a configured minimum. #[pyfunction] #[pyo3(signature = (*, severity, minimum))] -fn event_export_severity_at_least( +pub(crate) fn event_export_severity_at_least( severity: &Bound<'_, PyAny>, minimum: &Bound<'_, PyAny>, ) -> PyResult { @@ -49,16 +47,10 @@ fn event_export_severity_at_least( /// The stricter of a required and an optional minimum severity, as `EventExportSeverity`. #[pyfunction] #[pyo3(signature = (*, first, second))] -fn stricter_event_export_severity<'py>( +pub(crate) fn stricter_event_export_severity<'py>( first: &Bound<'py, PyAny>, second: Option<&Bound<'py, PyAny>>, ) -> PyResult> { let second: Option = second.map(severity_from).transpose()?; severity_object(first.py(), stricter_severity(severity_from(first)?, second)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(lifecycle_export_policy_catalog, module)?)?; - module.add_function(wrap_pyfunction!(event_export_severity_at_least, module)?)?; - module.add_function(wrap_pyfunction!(stricter_event_export_severity, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs b/crates/sqlbuild-python-output/src/event_exporting/_helpers/workers.rs similarity index 62% rename from crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs rename to crates/sqlbuild-python-output/src/event_exporting/_helpers/workers.rs index f2c3a7d1e9..8d78b78057 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs +++ b/crates/sqlbuild-python-output/src/event_exporting/_helpers/workers.rs @@ -4,28 +4,29 @@ use std::collections::VecDeque; use std::sync::Arc; use std::time::{Duration, Instant}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyclass, pymethods}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; use sqlbuild_core::event_export::constants::TIMEOUT_ERROR_TYPE; use sqlbuild_core::event_export::main::drop_lifecycle_attempts::drop_lifecycle_attempts; use sqlbuild_core::event_export::main::retain_outcome_events::retain_outcome_events; use sqlbuild_core::event_export::main::take_lifecycle_event::take_lifecycle_event; -use sqlbuild_core::event_export::models::{EventQueue, ExportSnapshot, QueuedEvent}; +use sqlbuild_core::event_export::models::{EventQueue, ExportSnapshot}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::event_exporting::dispatcher::{Exporter, Queued, Shared, State}; +use crate::event_exporting::_helpers::dispatcher::Exporter; +use crate::event_exporting::_helpers::dispatcher::Queued; +use crate::event_exporting::_helpers::dispatcher::Shared; +use crate::event_exporting::_helpers::dispatcher::State; +use crate::recall::main::write_stderr::write_stderr; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::event_exporting::constants::CHARGE_LIMIT_SECONDS; use sqlbuild_python_base::event_exporting::constants::INVOCATION_THREAD_PREFIX; use sqlbuild_python_base::event_exporting::constants::LANE_WINDOW; -use sqlbuild_python_base::event_exporting::constants::NO_SECONDS; -use sqlbuild_python_base::event_exporting::constants::POLL_SECONDS; use sqlbuild_python_base::event_exporting::main::error_type_of::error_type_of; use sqlbuild_python_base::event_exporting::models::EventExportSummary; use sqlbuild_python_base::event_exporting::models::EventExporterFailure; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_output::recall::main::write_stderr::write_stderr; /// Which long-lived dispatcher thread a worker runs. #[derive(Clone, Copy)] @@ -35,17 +36,11 @@ pub(crate) enum WorkerRole { Health, } -/// A long-lived dispatcher thread's target. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct DispatcherWorker { - shared: Arc, - role: WorkerRole, -} - /// Outcome events held for blocked exporters, each with its exporter index. type HeldOutcomes = Vec<(usize, Queued)>; /// One attempt handed to a lane and not yet reported. +#[derive(Debug)] pub(crate) struct Attempt { token: u64, queued: Queued, @@ -53,6 +48,7 @@ pub(crate) struct Attempt { } /// One exporter's long-lived delivery thread, its call queue and the attempts it still owes. +#[derive(Debug)] pub(crate) struct Lane { calls: Py, thread: Py, @@ -64,12 +60,12 @@ pub(crate) struct Lane { /// What one dispatch round reads from the dispatcher besides its state. #[derive(Clone, Copy)] pub(crate) struct Round<'a> { - shared: &'a Shared, - diagnostics: Option, + pub(crate) shared: &'a Shared, + pub(crate) diagnostics: Option, } /// The next thing the notification thread delivers. -enum Notification { +pub(crate) enum Notification { Periodic(ExportSnapshot), Final(Py), Failure(EventExporterFailure), @@ -123,7 +119,7 @@ impl Lane { impl State { /// Account reported attempts against their lanes, returning whether a diagnostic was queued. - fn record_reports( + pub(crate) fn record_reports( &mut self, py: Python<'_>, reports: Vec<(u64, Option)>, @@ -213,7 +209,7 @@ impl State { } /// Retire `index`'s lane: claimed calls fail as timed out and block it until they return. - fn abandon_lane( + pub(crate) fn abandon_lane( &mut self, py: Python<'_>, (index, timed_out): (usize, bool), @@ -259,7 +255,7 @@ impl State { } /// Abandon lanes whose started call outlived `timeout`, charging each round at most `CHARGE_LIMIT_SECONDS`. - fn expire_lanes( + pub(crate) fn expire_lanes( &mut self, py: Python<'_>, timeout: Duration, @@ -293,7 +289,11 @@ impl State { } /// Drop every queued and lane-waiting event that is not an outcome, counting each attempt. - fn reserve_for_outcomes(&mut self, py: Python<'_>, claims: &Py) -> PyResult<()> { + pub(crate) fn reserve_for_outcomes( + &mut self, + py: Python<'_>, + claims: &Py, + ) -> PyResult<()> { let capacity: usize = self.queue.capacity; let (queue, dropped) = retain_outcome_events(std::mem::replace( &mut self.queue, @@ -323,7 +323,7 @@ impl State { } /// Move queued events onto exporter lanes while every eligible lane has room. - fn feed(&mut self, py: Python<'_>, round: Round<'_>) -> PyResult { + pub(crate) fn feed(&mut self, py: Python<'_>, round: Round<'_>) -> PyResult { let mut notify: bool = false; while !self.queue.items.is_empty() { let capacity: usize = self.queue.capacity; @@ -422,7 +422,7 @@ impl State { } /// Whether any lane still owes a reported attempt. - fn busy(&self) -> bool { + pub(crate) fn busy(&self) -> bool { self.lanes .iter() .flatten() @@ -430,7 +430,7 @@ impl State { } /// End every lane once it has run what was queued before, keeping its thread to join. - fn retire_lanes(&mut self, py: Python<'_>) -> PyResult<()> { + pub(crate) fn retire_lanes(&mut self, py: Python<'_>) -> PyResult<()> { let lanes: Vec = self.lanes.drain(..).flatten().collect(); for lane in lanes { lane.retire(py)?; @@ -454,7 +454,7 @@ fn cancel(claims: &Bound<'_, PyDict>, token: u64) -> PyResult<()> { } /// Warn once per stall that a sink call outlived its timeout; best-effort, ignoring write errors. -fn warn_blocked(py: Python<'_>, name: &str, timeout: Duration) { +pub(crate) fn warn_blocked(py: Python<'_>, name: &str, timeout: Duration) { drop(write_stderr( py, &format!( @@ -464,230 +464,3 @@ fn warn_blocked(py: Python<'_>, name: &str, timeout: Duration) { ), )); } - -impl DispatcherWorker { - pub(crate) fn new(shared: Arc, role: WorkerRole) -> Self { - Self { shared, role } - } - - /// The state this worker's dispatcher shares. - pub(crate) fn shared(&self) -> &Shared { - &self.shared - } - - /// Wait until bound, returning `false` when stopped first. - fn await_bound(&self, py: Python<'_>) -> PyResult { - let shared = &self.shared; - let _held = shared.hold(py)?; - loop { - let state = shared.locked()?; - if state.bound { - return Ok(true); - } - if state.stopping || state.force_stop { - return Ok(false); - } - drop(state); - shared.wait(py, POLL_SECONDS)?; - } - } - - fn run_dispatch(&self, py: Python<'_>) -> PyResult<()> { - let result = (|| -> PyResult<()> { - if !self.await_bound(py)? { - return Ok(()); - } - while self.dispatch_round(py)? {} - Ok(()) - })(); - let retired: PyResult<()> = (|| { - let _held = self.shared.hold(py)?; - self.shared.locked()?.retire_lanes(py) - })(); - self.shared.dispatcher_finished(py)?; - retired?; - result - } - - /// Record reported outcomes, expire, trim and feed lanes, then wait; `false` once delivery stops. - fn dispatch_round(&self, py: Python<'_>) -> PyResult { - let shared = &self.shared; - let round = Round { - shared, - diagnostics: shared.diagnostics(), - }; - let _held = shared.hold(py)?; - let reports: Vec<(u64, Option)> = self.reported(py)?; - let mut state = shared.locked()?; - let mut notify: bool = state.record_reports(py, reports, round)?; - let stopped: bool = if Shared::past_deadline(&state) || state.force_stop { - let drained: Vec> = std::mem::take(&mut state.queue.items); - for item in &drained { - state.counts = drop_lifecycle_attempts( - std::mem::take(&mut state.counts), - &item.eligible_exporters, - ); - } - for index in 0..state.lanes.len() { - notify |= state.abandon_lane(py, (index, false), round)?; - } - for (index, _) in std::mem::take(&mut state.held) { - state.counts = drop_lifecycle_attempts(std::mem::take(&mut state.counts), &[index]); - } - drop(drained); - true - } else { - notify |= state.expire_lanes(py, shared.invocation_timeout, round)?; - if !state.reserved && state.reserve_at.is_some_and(|at| Instant::now() >= at) { - state.reserved = true; - state.reserve_for_outcomes(py, &shared.claims)?; - } - notify |= state.feed(py, round)?; - state.stopping && !state.busy() && state.queue.items.is_empty() && state.held.is_empty() - }; - let blocked: Vec = std::mem::take(&mut state.newly_blocked); - drop(state); - for name in blocked { - warn_blocked(py, &name, shared.invocation_timeout); - } - if notify { - shared.notify_all(py)?; - } - if !stopped { - shared.wait(py, POLL_SECONDS)?; - } - Ok(!stopped) - } - - /// Every `(token, sanitized error type)` delivery threads reported since the last round. - fn reported(&self, py: Python<'_>) -> PyResult)>> { - let results: &Bound<'_, PyAny> = self.shared.results.bind(py); - let mut reports: Vec<(u64, Option)> = Vec::new(); - while !results.call_method0("empty")?.is_truthy()? { - let (token, error): (u64, Bound<'_, PyAny>) = - results.call_method0("get_nowait")?.extract()?; - let error_type: Option = - (!error.is_none()).then(|| error_type_of(py, &PyErr::from_value(error))); - reports.push((token, error_type)); - } - Ok(reports) - } - - fn next_notification(&self, py: Python<'_>) -> PyResult { - let shared = &self.shared; - let _held = shared.hold(py)?; - loop { - let mut state = shared.locked()?; - if let Some(snapshot) = state.pending_periodic.take() { - state.periodic_inflight = Some(snapshot.health_signature()); - return Ok(Notification::Periodic(snapshot)); - } - state.periodic_inflight = None; - if !state.final_delivered - && let Some(summary) = &state.final_notification - { - return Ok(Notification::Final(summary.clone_ref(py))); - } - if let Some(failure) = state.failures.pop_front() { - return Ok(Notification::Failure(failure)); - } - if state.notification_stopping { - return Ok(Notification::Stop); - } - drop(state); - shared.wait(py, POLL_SECONDS)?; - } - } - - fn run_notifications(&self, py: Python<'_>) -> PyResult<()> { - let shared = &self.shared; - loop { - match self.next_notification(py)? { - Notification::Periodic(snapshot) => { - let signature: [u64; 7] = snapshot.health_signature(); - if let Some(callback) = &shared.summary_callback { - let summary = EventExportSummary::object(py, snapshot)?; - drop(callback.bind(py).call1((summary,))); - } - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - state.last_published_periodic = signature; - state.periodic_inflight = None; - } - Notification::Final(summary) => { - if let Some(callback) = &shared.summary_callback { - drop(callback.bind(py).call1((summary,))); - } - let released = { - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - state.final_delivered = true; - state.final_notification.take() - }; - drop(released); - return Ok(()); - } - Notification::Failure(failure) => { - if let Some(callback) = &shared.failure_callback { - let failure = Py::new(py, failure)?; - drop(callback.bind(py).call1((failure,))); - } - } - Notification::Stop => return Ok(()), - } - } - } - - /// Wait one health interval, returning `false` once health publication stops. - fn await_interval(&self, py: Python<'_>) -> PyResult { - let shared = &self.shared; - let deadline: Instant = Instant::now() + shared.health_interval; - let _held = shared.hold(py)?; - loop { - if shared.locked()?.health_stopping { - return Ok(false); - } - let remaining: f64 = deadline - .saturating_duration_since(Instant::now()) - .as_secs_f64(); - if remaining <= NO_SECONDS { - return Ok(true); - } - shared.wait(py, remaining)?; - } - } - - fn run_health(&self, py: Python<'_>) -> PyResult<()> { - let shared = &self.shared; - while self.await_interval(py)? { - let snapshot: ExportSnapshot = shared.current_snapshot(py)?; - let signature: [u64; 7] = snapshot.health_signature(); - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - if state.health_stopping || state.final_notification.is_some() { - return Ok(()); - } - if signature == state.last_published_periodic - || state.periodic_inflight == Some(signature) - { - state.pending_periodic = None; - continue; - } - state.pending_periodic = Some(snapshot); - drop(state); - shared.notify_all(py)?; - } - Ok(()) - } -} - -#[pymethods] -impl DispatcherWorker { - fn __call__(&self, py: Python<'_>) -> PyResult<()> { - match self.role { - WorkerRole::Dispatch => self.run_dispatch(py), - WorkerRole::Notify => self.run_notifications(py), - WorkerRole::Health => self.run_health(py), - } - } -} diff --git a/crates/sqlbuild-python-output/src/event_exporting/constants.rs b/crates/sqlbuild-python-output/src/event_exporting/constants.rs new file mode 100644 index 0000000000..c74a8cb0b7 --- /dev/null +++ b/crates/sqlbuild-python-output/src/event_exporting/constants.rs @@ -0,0 +1,50 @@ +//! Public constants of event exporting. + +use crate::event_exporting::_helpers::getters::read_bound_event_exporter; +use crate::event_exporting::_helpers::getters::read_event_export_summary; +use crate::event_exporting::_helpers::getters::read_event_exporter_accounting; +use crate::event_exporting::_helpers::getters::read_event_exporter_command_scope; +use crate::event_exporting::_helpers::getters::read_event_exporter_counts; +use crate::event_exporting::_helpers::getters::read_event_exporter_dispatcher; +use crate::event_exporting::_helpers::getters::read_event_exporter_failure; +use crate::event_exporting::models::EventExporterDispatcher; +use sqlbuild_python_base::event_exporting::models::BoundEventExporter; +use sqlbuild_python_base::event_exporting::models::EventExportSummary; +use sqlbuild_python_base::event_exporting::models::EventExporterAccounting; +use sqlbuild_python_base::event_exporting::models::EventExporterCommandScope; +use sqlbuild_python_base::event_exporting::models::EventExporterCounts; +use sqlbuild_python_base::event_exporting::models::EventExporterFailure; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_event_exporter_dispatcher, + }, + GetterEntry { + class: class_of::, + read: read_bound_event_exporter, + }, + GetterEntry { + class: class_of::, + read: read_event_export_summary, + }, + GetterEntry { + class: class_of::, + read: read_event_exporter_accounting, + }, + GetterEntry { + class: class_of::, + read: read_event_exporter_counts, + }, + GetterEntry { + class: class_of::, + read: read_event_exporter_failure, + }, + GetterEntry { + class: class_of::, + read: read_event_exporter_command_scope, + }, +]; diff --git a/crates/sqlbuild-python-output/src/event_exporting/main/mod.rs b/crates/sqlbuild-python-output/src/event_exporting/main/mod.rs new file mode 100644 index 0000000000..72044fb874 --- /dev/null +++ b/crates/sqlbuild-python-output/src/event_exporting/main/mod.rs @@ -0,0 +1,2 @@ +pub mod register_policy; +pub mod stop_event_exporter_threads_function; diff --git a/crates/sqlbuild-python-output/src/event_exporting/main/register_policy.rs b/crates/sqlbuild-python-output/src/event_exporting/main/register_policy.rs new file mode 100644 index 0000000000..65ef310ab1 --- /dev/null +++ b/crates/sqlbuild-python-output/src/event_exporting/main/register_policy.rs @@ -0,0 +1,13 @@ +//! Register the policy functions and classes. + +use crate::event_exporting::_helpers::policy::event_export_severity_at_least; +use crate::event_exporting::_helpers::policy::lifecycle_export_policy_catalog; +use crate::event_exporting::_helpers::policy::stricter_event_export_severity; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, wrap_pyfunction}; + +/// Register the policy functions and classes. +pub fn register_policy(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(lifecycle_export_policy_catalog, module)?)?; + module.add_function(wrap_pyfunction!(event_export_severity_at_least, module)?)?; + module.add_function(wrap_pyfunction!(stricter_event_export_severity, module)?) +} diff --git a/crates/sqlbuild-python-output/src/event_exporting/main/stop_event_exporter_threads_function.rs b/crates/sqlbuild-python-output/src/event_exporting/main/stop_event_exporter_threads_function.rs new file mode 100644 index 0000000000..254346fa44 --- /dev/null +++ b/crates/sqlbuild-python-output/src/event_exporting/main/stop_event_exporter_threads_function.rs @@ -0,0 +1,10 @@ +//! `stop_event_exporter_threads` as a function of `module`. + +use crate::event_exporting::_helpers::dispatcher::stop_event_exporter_threads; + +/// `stop_event_exporter_threads` as a function of `module`. +pub fn stop_event_exporter_threads_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(stop_event_exporter_threads, module) +} diff --git a/crates/sqlbuild-python-output/src/event_exporting/mod.rs b/crates/sqlbuild-python-output/src/event_exporting/mod.rs new file mode 100644 index 0000000000..0070ad7544 --- /dev/null +++ b/crates/sqlbuild-python-output/src/event_exporting/mod.rs @@ -0,0 +1,6 @@ +//! Event exporting: the dispatcher that queues build events, the delivery workers and the export policy. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-output/src/event_exporting/models.rs b/crates/sqlbuild-python-output/src/event_exporting/models.rs new file mode 100644 index 0000000000..e8b23c67bd --- /dev/null +++ b/crates/sqlbuild-python-output/src/event_exporting/models.rs @@ -0,0 +1,691 @@ +//! Public records of event exporting. + +use crate::event_exporting::_helpers::dispatcher::Delivery; +use crate::event_exporting::_helpers::dispatcher::Limits; +use crate::event_exporting::_helpers::dispatcher::OutputSink; +use crate::event_exporting::_helpers::dispatcher::Queued; +use crate::event_exporting::_helpers::dispatcher::Shared; +use crate::event_exporting::_helpers::dispatcher::State; +use crate::event_exporting::_helpers::dispatcher::join; +use crate::event_exporting::_helpers::dispatcher::limits; +use crate::event_exporting::_helpers::dispatcher::read_exporters; +use crate::event_exporting::_helpers::dispatcher::read_output_sinks; +use crate::event_exporting::_helpers::dispatcher::remaining_seconds; +use crate::event_exporting::_helpers::dispatcher::retire_output_lane; +use crate::event_exporting::_helpers::dispatcher::start_worker; +use crate::event_exporting::_helpers::workers::Notification; +use crate::event_exporting::_helpers::workers::Round; +use crate::event_exporting::_helpers::workers::WorkerRole; +use crate::event_exporting::_helpers::workers::warn_blocked; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::prelude::pyclass; +use pyo3::prelude::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyFloat; +use sqlbuild_core::command_output::constants::COMMAND_OUTPUT_RECORD_TYPE; +use sqlbuild_core::event_export::constants::INVOCATION_START_EVENT_TYPE; +use sqlbuild_core::event_export::constants::INVOCATION_TERMINAL_EVENT_TYPES; +use sqlbuild_core::event_export::constants::OUTCOME_EVENT_TYPES; +use sqlbuild_core::event_export::constants::OUTCOME_RESERVE_FRACTION; +use sqlbuild_core::event_export::main::drop_lifecycle_attempts::drop_lifecycle_attempts; +use sqlbuild_core::event_export::main::queue_lifecycle_event::queue_lifecycle_event; +use sqlbuild_core::event_export::main::route_lifecycle_event::route_lifecycle_event; +use sqlbuild_core::event_export::models::EventQueue; +use sqlbuild_core::event_export::models::ExportPolicy; +use sqlbuild_core::event_export::models::ExportSnapshot; +use sqlbuild_core::event_export::models::ExporterCounts; +use sqlbuild_core::event_export::models::QueuedEvent; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::event_exporting::constants::DISPATCHER_THREAD_NAME; +use sqlbuild_python_base::event_exporting::constants::EXIT_JOIN_SECONDS; +use sqlbuild_python_base::event_exporting::constants::HEALTH_THREAD_NAME; +use sqlbuild_python_base::event_exporting::constants::JOIN_GRACE_SECONDS; +use sqlbuild_python_base::event_exporting::constants::NO_SECONDS; +use sqlbuild_python_base::event_exporting::constants::NOTIFIER_THREAD_NAME; +use sqlbuild_python_base::event_exporting::constants::POLL_SECONDS; +use sqlbuild_python_base::event_exporting::main::error_type_of::error_type_of; +use sqlbuild_python_base::event_exporting::main::export_policy::export_policy; +use sqlbuild_python_base::event_exporting::main::state_error::state_error; +use sqlbuild_python_base::event_exporting::models::EventExportSummary; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::HashMap; +use std::collections::HashSet; +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Instant; + +/// Deliver filtered lifecycle events off execution threads with bounded memory. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct EventExporterDispatcher { + pub(crate) shared: Arc, + /// Event kinds this command never exports, such as compile's phase operations. + pub(crate) withheld_kinds: Vec, + pub(crate) thread: Py, + pub(crate) notification_thread: Option>, + pub(crate) health_thread: Option>, +} + +#[pymethods] +impl EventExporterDispatcher { + #[new] + #[pyo3(signature = (**options))] + pub fn new(py: Python<'_>, options: Option>) -> PyResult { + let options: KeywordArguments<'_> = KeywordArguments::read( + py, + "EventExporterDispatcher", + options, + ( + &[], + &[ + "exporters", + "queue_capacity", + "shutdown_timeout_seconds", + "invocation_timeout_seconds", + "notification_queue_capacity", + "health_interval_seconds", + "failure_callback", + "summary_callback", + "withheld_kinds", + ], + ), + )?; + let withheld: Bound<'_, PyAny> = options.value("withheld_kinds")?; + let withheld_kinds: Vec = if withheld.is_none() { + Vec::new() + } else { + withheld.extract()? + }; + let limits: Limits = limits(&options)?; + let exporters: Bound<'_, PyAny> = options.value("exporters")?; + let (bound_exporters, filters) = if exporters.is_none() { + (Vec::new(), Vec::new()) + } else { + read_exporters(&exporters)? + }; + let callback = |name: &str| -> PyResult>> { + let value: Bound<'_, PyAny> = options.value(name)?; + Ok((!value.is_none()).then(|| value.unbind())) + }; + let failure_callback: Option> = callback("failure_callback")?; + let summary_callback: Option> = callback("summary_callback")?; + let notifications: bool = failure_callback.is_some() || summary_callback.is_some(); + let has_summary: bool = summary_callback.is_some(); + let shared = Arc::new(Shared { + condition: imported(py, "threading", "Condition")?.call0()?.unbind(), + results: imported(py, "queue", "SimpleQueue")?.call0()?.unbind(), + claims: PyDict::new(py).unbind(), + state: Mutex::new(State { + bound: !exporters.is_none(), + counts: vec![ExporterCounts::default(); bound_exporters.len()], + exporters: bound_exporters, + filters, + output_sinks: Arc::new(Vec::new()), + output_lane: None, + queue: EventQueue::new(limits.queue_capacity), + sequence: 0, + accepting: true, + deadline: None, + reserve_at: None, + reserved: false, + lanes: Vec::new(), + tokens: HashMap::new(), + next_token: 0, + retired: Vec::new(), + abandoned_calls: false, + stopping: false, + force_stop: false, + blocked: HashSet::new(), + stalled: HashMap::new(), + held: Vec::new(), + newly_blocked: Vec::new(), + last_charged: None, + live_invocations: HashSet::new(), + next_invocation: 0, + dispatcher_running: true, + shutdown_started: false, + shutdown_complete: false, + final_summary: None, + failures: VecDeque::new(), + notification_stopping: false, + health_stopping: false, + pending_periodic: None, + periodic_inflight: None, + last_published_periodic: [ + 0, + 0, + 0, + 0, + 0, + 0, + u64::try_from(limits.queue_capacity).unwrap_or(u64::MAX), + ], + final_notification: None, + final_delivered: false, + }), + invocation_timeout: limits.invocation_timeout, + shutdown_timeout: limits.shutdown_timeout, + health_interval: limits.health_interval, + notification_capacity: limits.notification_capacity, + failure_callback, + summary_callback, + notifications, + }); + let notification_thread: Option> = notifications + .then(|| start_worker(py, &shared, (WorkerRole::Notify, NOTIFIER_THREAD_NAME))) + .transpose()?; + let health_thread: Option> = has_summary + .then(|| start_worker(py, &shared, (WorkerRole::Health, HEALTH_THREAD_NAME))) + .transpose()?; + for thread in notification_thread.iter().chain(&health_thread) { + thread.call_method0("start")?; + } + let thread = start_worker(py, &shared, (WorkerRole::Dispatch, DISPATCHER_THREAD_NAME))?; + thread.call_method0("start")?; + Ok(Self { + shared, + withheld_kinds, + thread: thread.unbind(), + notification_thread: notification_thread.map(Bound::unbind), + health_thread: health_thread.map(Bound::unbind), + }) + } + + #[getter] + pub fn thread(&self, py: Python<'_>) -> Py { + self.thread.clone_ref(py) + } + + #[getter] + pub fn notification_thread(&self, py: Python<'_>) -> Option> { + self.notification_thread + .as_ref() + .map(|thread| thread.clone_ref(py)) + } + + /// Filter and enqueue eligible exporter attempts without waiting. + pub(crate) fn enqueue(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { + let event_type: String = native_field(event, "event_type")?.extract()?; + let policy: ExportPolicy = export_policy(py, &event_type)?; + if self.withheld_kinds.contains(&policy.kind) { + return Ok(()); + } + let invocation_id: Option = match native_field(event, "invocation_id") { + Ok(value) => value.extract()?, + Err(_) => None, + }; + let shared = &self.shared; + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + if !state.bound { + drop(state); + return Err(state_error( + py, + "event exporters must be bound before enqueue", + )); + } + let counts: Vec = std::mem::take(&mut state.counts); + let (counts, eligible) = route_lifecycle_event(&policy, &state.filters, counts); + state.counts = counts; + if eligible.is_empty() { + return Ok(()); + } + if !state.accepting { + state.counts = drop_lifecycle_attempts(std::mem::take(&mut state.counts), &eligible); + return Ok(()); + } + let sequence: u64 = state.sequence; + state.sequence += 1; + let capacity: usize = state.queue.capacity; + let mut outcome = queue_lifecycle_event( + std::mem::replace(&mut state.queue, EventQueue::new(capacity)), + QueuedEvent { + sequence, + item: Queued { + event: event.clone().unbind(), + kind: policy.kind, + severity: policy.severity, + }, + priority: policy.priority, + invocation_terminal: INVOCATION_TERMINAL_EVENT_TYPES.contains(&event_type.as_str()), + outcome: OUTCOME_EVENT_TYPES.contains(&event_type.as_str()), + invocation_id, + invocation_start: event_type == INVOCATION_START_EVENT_TYPE, + eligible_exporters: eligible, + }, + ); + state.queue = std::mem::replace(&mut outcome.queue, EventQueue::new(capacity)); + for lost in outcome.displaced.iter().chain(&outcome.rejected) { + state.counts = drop_lifecycle_attempts( + std::mem::take(&mut state.counts), + &lost.eligible_exporters, + ); + } + drop(state); + shared.notify_all(py)?; + drop(outcome); + Ok(()) + } + + /// Bind validated exporters once before lifecycle publication starts. + pub(crate) fn bind(&self, py: Python<'_>, exporters: &Bound<'_, PyAny>) -> PyResult<()> { + let (exporters, filters) = read_exporters(exporters)?; + let shared = &self.shared; + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + if state.bound { + drop(state); + return Err(state_error(py, "event exporters are already bound")); + } + state.counts = vec![ExporterCounts::default(); exporters.len()]; + state.exporters = exporters; + state.filters = filters; + state.bound = true; + drop(state); + shared.notify_all(py) + } + + /// Bind explicit command-output sinks before capture starts. + pub(crate) fn bind_command_output_sinks( + &self, + py: Python<'_>, + sinks: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let sinks: Arc> = Arc::new(read_output_sinks(sinks)?); + let previous = { + let _held = self.shared.hold(py)?; + std::mem::replace(&mut self.shared.locked()?.output_sinks, sinks) + }; + drop(previous); + Ok(()) + } + + /// Deliver output records through the already-bound destination providers. + pub(crate) fn export_output(&self, py: Python<'_>, records: &Bound<'_, PyAny>) -> PyResult<()> { + let shared = &self.shared; + let invocation: u64 = shared.begin_invocation(py)?; + let sinks: Arc> = { + let _held = shared.hold(py)?; + Arc::clone(&shared.locked()?.output_sinks) + }; + let mut failed: bool = false; + let delivered = (|| -> PyResult { + for record in records.try_iter()? { + let record: Bound<'_, PyAny> = record?; + let is_output: bool = + native_field(&record, "record_type")?.eq(COMMAND_OUTPUT_RECORD_TYPE)?; + let stream: Option = if is_output { + Some(native_field(&record, "stream")?.extract()?) + } else { + None + }; + for sink in sinks.iter() { + if stream + .as_ref() + .is_some_and(|stream| !sink.streams.contains(stream)) + { + continue; + } + match shared.call_output_sink(py, sink, &record)? { + Delivery::Delivered(sink_failed) => failed |= sink_failed, + Delivery::Abandoned => return Ok(Delivery::Abandoned), + } + } + } + Ok(Delivery::Delivered(failed)) + })(); + if let Ok(Delivery::Abandoned) = delivered { + return Err(state_error( + py, + "output export stopped because the interpreter is exiting", + )); + } + shared.invocation_finished(py, invocation)?; + delivered?; + if failed { + return Err(state_error(py, "one or more output export attempts failed")); + } + Ok(()) + } + + /// Stop acceptance and drain until the fixed shutdown deadline. + pub(crate) fn shutdown(&self, py: Python<'_>) -> PyResult> { + let shared = &self.shared; + let (owns, deadline) = { + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + let owns: bool = !state.shutdown_started; + if owns { + state.shutdown_started = true; + state.accepting = false; + let started: Instant = Instant::now(); + state.deadline = Some(started + shared.shutdown_timeout); + state.reserve_at = Some( + started + + shared + .shutdown_timeout + .mul_f64(1.0 - OUTCOME_RESERVE_FRACTION), + ); + state.stopping = true; + state.health_stopping = true; + } + let deadline: Instant = state.deadline.unwrap_or_else(Instant::now); + drop(state); + shared.notify_all(py)?; + (owns, deadline) + }; + if !owns { + return shared.await_shutdown(py); + } + let thread: &Bound<'_, PyAny> = self.thread.bind(py); + call_with( + &native_field(thread, "join")?, + &[( + "timeout", + PyFloat::new(py, remaining_seconds(deadline) + JOIN_GRACE_SECONDS).into_any(), + )], + )?; + if thread.call_method0("is_alive")?.is_truthy()? { + { + let _held = shared.hold(py)?; + shared.locked()?.force_stop = true; + shared.notify_all(py)?; + } + thread.call_method0("join")?; + } + let snapshot: ExportSnapshot = { + let _held = shared.hold(py)?; + Shared::snapshot(&*shared.locked()?, true) + }; + let summary: Py = EventExportSummary::object(py, snapshot)?; + { + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + state.final_summary = Some(summary.clone_ref(py)); + state.shutdown_complete = true; + if shared.summary_callback.is_some() + && state.final_notification.is_none() + && !state.final_delivered + { + state.final_notification = Some(summary.clone_ref(py)); + } + state.notification_stopping = true; + let retired: Option> = if state.live_invocations.is_empty() { + state.output_lane.take() + } else { + None + }; + drop(state); + shared.notify_all(py)?; + retire_output_lane(py, retired)?; + } + if let Some(health) = &self.health_thread { + join(health.bind(py), EXIT_JOIN_SECONDS)?; + } + Ok(summary) + } + + /// Run `finalizer` on the calling thread, never a daemon delivery thread, once delivery is idle or the idle wait ends. + pub(crate) fn finalize_when_idle(&self, py: Python<'_>, finalizer: Py) -> PyResult<()> { + self.shared.await_idle(py)?; + drop(finalizer.bind(py).call0()); + Ok(()) + } + + /// Return a thread-safe immutable point-in-time snapshot. + pub(crate) fn summary(&self, py: Python<'_>) -> PyResult> { + let snapshot: ExportSnapshot = self.shared.current_snapshot(py)?; + EventExportSummary::object(py, snapshot) + } +} + +/// A long-lived dispatcher thread's target. +#[pyclass(module = "sqlbuild._native", frozen)] +pub struct DispatcherWorker { + pub(crate) shared: Arc, + pub(crate) role: WorkerRole, +} + +impl std::fmt::Debug for DispatcherWorker { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("DispatcherWorker") + .finish_non_exhaustive() + } +} + +impl DispatcherWorker { + pub(crate) fn new(shared: Arc, role: WorkerRole) -> Self { + Self { shared, role } + } + + /// The state this worker's dispatcher shares. + pub(crate) fn shared(&self) -> &Shared { + &self.shared + } + + /// Wait until bound, returning `false` when stopped first. + pub(crate) fn await_bound(&self, py: Python<'_>) -> PyResult { + let shared = &self.shared; + let _held = shared.hold(py)?; + loop { + let state = shared.locked()?; + if state.bound { + return Ok(true); + } + if state.stopping || state.force_stop { + return Ok(false); + } + drop(state); + shared.wait(py, POLL_SECONDS)?; + } + } + + pub(crate) fn run_dispatch(&self, py: Python<'_>) -> PyResult<()> { + let result = (|| -> PyResult<()> { + if !self.await_bound(py)? { + return Ok(()); + } + while self.dispatch_round(py)? {} + Ok(()) + })(); + let retired: PyResult<()> = (|| { + let _held = self.shared.hold(py)?; + self.shared.locked()?.retire_lanes(py) + })(); + self.shared.dispatcher_finished(py)?; + retired?; + result + } + + /// Record reported outcomes, expire, trim and feed lanes, then wait; `false` once delivery stops. + pub(crate) fn dispatch_round(&self, py: Python<'_>) -> PyResult { + let shared = &self.shared; + let round = Round { + shared, + diagnostics: shared.diagnostics(), + }; + let _held = shared.hold(py)?; + let reports: Vec<(u64, Option)> = self.reported(py)?; + let mut state = shared.locked()?; + let mut notify: bool = state.record_reports(py, reports, round)?; + let stopped: bool = if Shared::past_deadline(&state) || state.force_stop { + let drained: Vec> = std::mem::take(&mut state.queue.items); + for item in &drained { + state.counts = drop_lifecycle_attempts( + std::mem::take(&mut state.counts), + &item.eligible_exporters, + ); + } + for index in 0..state.lanes.len() { + notify |= state.abandon_lane(py, (index, false), round)?; + } + for (index, _) in std::mem::take(&mut state.held) { + state.counts = drop_lifecycle_attempts(std::mem::take(&mut state.counts), &[index]); + } + drop(drained); + true + } else { + notify |= state.expire_lanes(py, shared.invocation_timeout, round)?; + if !state.reserved && state.reserve_at.is_some_and(|at| Instant::now() >= at) { + state.reserved = true; + state.reserve_for_outcomes(py, &shared.claims)?; + } + notify |= state.feed(py, round)?; + state.stopping && !state.busy() && state.queue.items.is_empty() && state.held.is_empty() + }; + let blocked: Vec = std::mem::take(&mut state.newly_blocked); + drop(state); + for name in blocked { + warn_blocked(py, &name, shared.invocation_timeout); + } + if notify { + shared.notify_all(py)?; + } + if !stopped { + shared.wait(py, POLL_SECONDS)?; + } + Ok(!stopped) + } + + /// Every `(token, sanitized error type)` delivery threads reported since the last round. + pub(crate) fn reported(&self, py: Python<'_>) -> PyResult)>> { + let results: &Bound<'_, PyAny> = self.shared.results.bind(py); + let mut reports: Vec<(u64, Option)> = Vec::new(); + while !results.call_method0("empty")?.is_truthy()? { + let (token, error): (u64, Bound<'_, PyAny>) = + results.call_method0("get_nowait")?.extract()?; + let error_type: Option = + (!error.is_none()).then(|| error_type_of(py, &PyErr::from_value(error))); + reports.push((token, error_type)); + } + Ok(reports) + } + + pub(crate) fn next_notification(&self, py: Python<'_>) -> PyResult { + let shared = &self.shared; + let _held = shared.hold(py)?; + loop { + let mut state = shared.locked()?; + if let Some(snapshot) = state.pending_periodic.take() { + state.periodic_inflight = Some(snapshot.health_signature()); + return Ok(Notification::Periodic(snapshot)); + } + state.periodic_inflight = None; + if !state.final_delivered + && let Some(summary) = &state.final_notification + { + return Ok(Notification::Final(summary.clone_ref(py))); + } + if let Some(failure) = state.failures.pop_front() { + return Ok(Notification::Failure(failure)); + } + if state.notification_stopping { + return Ok(Notification::Stop); + } + drop(state); + shared.wait(py, POLL_SECONDS)?; + } + } + + pub(crate) fn run_notifications(&self, py: Python<'_>) -> PyResult<()> { + let shared = &self.shared; + loop { + match self.next_notification(py)? { + Notification::Periodic(snapshot) => { + let signature: [u64; 7] = snapshot.health_signature(); + if let Some(callback) = &shared.summary_callback { + let summary = EventExportSummary::object(py, snapshot)?; + drop(callback.bind(py).call1((summary,))); + } + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + state.last_published_periodic = signature; + state.periodic_inflight = None; + } + Notification::Final(summary) => { + if let Some(callback) = &shared.summary_callback { + drop(callback.bind(py).call1((summary,))); + } + let released = { + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + state.final_delivered = true; + state.final_notification.take() + }; + drop(released); + return Ok(()); + } + Notification::Failure(failure) => { + if let Some(callback) = &shared.failure_callback { + let failure = Py::new(py, failure)?; + drop(callback.bind(py).call1((failure,))); + } + } + Notification::Stop => return Ok(()), + } + } + } + + /// Wait one health interval, returning `false` once health publication stops. + pub(crate) fn await_interval(&self, py: Python<'_>) -> PyResult { + let shared = &self.shared; + let deadline: Instant = Instant::now() + shared.health_interval; + let _held = shared.hold(py)?; + loop { + if shared.locked()?.health_stopping { + return Ok(false); + } + let remaining: f64 = deadline + .saturating_duration_since(Instant::now()) + .as_secs_f64(); + if remaining <= NO_SECONDS { + return Ok(true); + } + shared.wait(py, remaining)?; + } + } + + pub(crate) fn run_health(&self, py: Python<'_>) -> PyResult<()> { + let shared = &self.shared; + while self.await_interval(py)? { + let snapshot: ExportSnapshot = shared.current_snapshot(py)?; + let signature: [u64; 7] = snapshot.health_signature(); + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + if state.health_stopping || state.final_notification.is_some() { + return Ok(()); + } + if signature == state.last_published_periodic + || state.periodic_inflight == Some(signature) + { + state.pending_periodic = None; + continue; + } + state.pending_periodic = Some(snapshot); + drop(state); + shared.notify_all(py)?; + } + Ok(()) + } +} + +#[pymethods] +impl DispatcherWorker { + pub(crate) fn __call__(&self, py: Python<'_>) -> PyResult<()> { + match self.role { + WorkerRole::Dispatch => self.run_dispatch(py), + WorkerRole::Notify => self.run_notifications(py), + WorkerRole::Health => self.run_health(py), + } + } +} diff --git a/crates/sqlbuild-python-output/src/lib.rs b/crates/sqlbuild-python-output/src/lib.rs index 10d1540e1b..a3704154f9 100644 --- a/crates/sqlbuild-python-output/src/lib.rs +++ b/crates/sqlbuild-python-output/src/lib.rs @@ -5,6 +5,7 @@ pub mod build_progress; pub mod compute_logs; pub mod cost_telemetry; +pub mod event_exporting; pub mod output_capture; pub mod recall; pub mod result_output; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index 84d84dc90b..be8bef7517 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -15,7 +15,6 @@ use sqlbuild_cli::dispatch::models::{ExportSummary, ExporterLoss}; use sqlbuild_core::event_export::constants::DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::event_exporting::dispatcher::EventExporterDispatcher; use crate::bindings::_helpers::native_host::setup_context::invocation_setup_context; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -30,6 +29,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_host::main::redirect_stdout_to_stderr::redirect_stdout_to_stderr; use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_output::event_exporting::models::EventExporterDispatcher; use sqlbuild_python_output::recall::main::write_stderr::write_stderr; use sqlbuild_python_output::result_output::main::events::terminal_event_index_scope::terminal_event_index_scope; use sqlbuild_python_output::result_output::models::TerminalEventIndex; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/mod.rs index 16eba7bfb1..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/mod.rs @@ -1,5 +1 @@ -pub(crate) mod dispatcher; -pub(crate) mod getters; -pub(crate) mod policy; pub(crate) mod registration; -pub(crate) mod workers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs index 6335096ee9..0bbb7d92e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs @@ -1,12 +1,7 @@ //! Register native lifecycle-event export and stop its threads at interpreter exit. -use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use crate::bindings::_helpers::event_exporting::dispatcher::{ - EventExporterDispatcher, stop_event_exporter_threads, -}; -use crate::bindings::_helpers::event_exporting::policy; -use crate::bindings::_helpers::event_exporting::workers::DispatcherWorker; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::event_exporting::main::register_scope::register_scope; use sqlbuild_python_base::event_exporting::models::BoundEventExporter; @@ -14,6 +9,10 @@ use sqlbuild_python_base::event_exporting::models::EventExportSummary; use sqlbuild_python_base::event_exporting::models::EventExporterAccounting; use sqlbuild_python_base::event_exporting::models::EventExporterCounts; use sqlbuild_python_base::event_exporting::models::EventExporterFailure; +use sqlbuild_python_output::event_exporting::main::register_policy::register_policy; +use sqlbuild_python_output::event_exporting::main::stop_event_exporter_threads_function::stop_event_exporter_threads_function; +use sqlbuild_python_output::event_exporting::models::DispatcherWorker; +use sqlbuild_python_output::event_exporting::models::EventExporterDispatcher; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; @@ -23,9 +22,9 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_class::()?; module.add_class::()?; - let hook = wrap_pyfunction!(stop_event_exporter_threads, module)?; + let hook = stop_event_exporter_threads_function(module)?; module.add_function(hook.clone())?; imported(module.py(), "atexit", "register")?.call1((hook,))?; - policy::register(module)?; + register_policy(module)?; register_scope(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index d3a487ceda..d8e63e1987 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -68,7 +68,7 @@ use crate::bindings::_helpers::record_behaviours::scopes::{ use crate::bindings::_helpers::record_behaviours::values::{ finish_fingerprint_set, logical_type_display_name, sql_value_kind, }; -use crate::bindings::_helpers::{dbt_events, dbt_interop, event_exporting}; +use crate::bindings::_helpers::{dbt_events, dbt_interop}; use sqlbuild_python_base::records::models::{GetterEntry, RecordBehaviours}; use sqlbuild_python_base::records::types::ClassMethod; use sqlbuild_python_base::records::types::ClassProperty; @@ -364,7 +364,7 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ dbt_interop::getters::GETTERS, sqlbuild_python_base::diagnostics::constants::GETTERS, sqlbuild_python_front::discovery::constants::GETTERS, - event_exporting::getters::GETTERS, + sqlbuild_python_output::event_exporting::constants::GETTERS, sqlbuild_python_front::execution_history::constants::GETTERS, sqlbuild_python_project::graph::constants::GETTERS, sqlbuild_python_front::model_config::constants::GETTERS, diff --git a/fensu.toml b/fensu.toml index acb58027a7..f79f02003f 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1675,7 +1675,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 52134 } +thresholds = { max_crate_code_lines = 50333 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] @@ -1895,6 +1895,12 @@ path = "crates/sqlbuild-python-output/src/recall/models.rs" symbols = ["RecallReplacements"] reason = "RecallReplacements implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; other models derive it." +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-output/src/event_exporting/models.rs" +symbols = ["DispatcherWorker"] +reason = "DispatcherWorker implements Debug manually because its WorkerRole does not implement Debug; other models derive it." + [[targets.rust.rule_exceptions]] rule = "FPRSR307" path = "crates/sqlbuild-rules/src/scope_metadata/main.rs" @@ -2482,6 +2488,9 @@ pending_budgets = [ "crates/sqlbuild-python-output/src/cost_telemetry/_helpers/allocation.rs=1", "crates/sqlbuild-python-output/src/cost_telemetry/_helpers/ledger.rs=13", "crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs=23", + "crates/sqlbuild-python-output/src/event_exporting/_helpers/dispatcher.rs=19", + "crates/sqlbuild-python-output/src/event_exporting/_helpers/workers.rs=6", + "crates/sqlbuild-python-output/src/event_exporting/models.rs=14", "crates/sqlbuild-python-output/src/output_capture/_helpers/context.rs=5", "crates/sqlbuild-python-output/src/output_capture/_helpers/dispatcher.rs=29", "crates/sqlbuild-python-output/src/output_capture/_helpers/records.rs=13", @@ -2767,9 +2776,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs=1", @@ -2963,6 +2970,9 @@ pending_budgets = [ "crates/sqlbuild-python-output/src/build_progress/_helpers/nested_progress.rs=2", "crates/sqlbuild-python-output/src/build_progress/models.rs=18", "crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs=7", + "crates/sqlbuild-python-output/src/event_exporting/_helpers/dispatcher.rs=11", + "crates/sqlbuild-python-output/src/event_exporting/_helpers/workers.rs=2", + "crates/sqlbuild-python-output/src/event_exporting/models.rs=3", "crates/sqlbuild-python-output/src/output_capture/_helpers/context.rs=2", "crates/sqlbuild-python-output/src/output_capture/_helpers/dispatcher.rs=9", "crates/sqlbuild-python-output/src/recall/_helpers/replacements.rs=3", @@ -3053,8 +3063,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs=6", @@ -3127,7 +3135,7 @@ module_layers = [ "diagnostics=registry", "diff_report=commands", "discovery=registry", - "event_exporting=output", + "event_exporting=registry", "execution_history=registry", "exported_constants=registry", "fingerprints=registry", @@ -3409,7 +3417,7 @@ paths = ["crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs" reason = "Each public entry forwards to its own helper with the same arguments." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs", "crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3685,7 +3693,7 @@ paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/oracles.rs", "crates reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3789,7 +3797,7 @@ paths = ["crates/sqlbuild-python-compile/src/compile_driver/main/stages/register reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_report.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_project.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_python_plan_entries.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_report.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_project.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_python_plan_entries.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3980,6 +3988,14 @@ reason = "Each progress reporter runs its own spinner loop and draws its own lin paths = ["crates/sqlbuild-python-output/src/build_progress/models.rs"] reason = "Each reporter clears its own transient line under its own lock; sharing one line writer between the reporters is a separate change." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_report.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_project.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_python_plan_entries.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-output/src/event_exporting/main/register_policy.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + [[dupes.allowlist]] paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs"] reason = "Each is its own Python entry point with its own keyword contract and dbt command shape; both end in the shared argv builder." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-output/src/event_exporting/_helpers/getters.rs"] +reason = "The export summary and the per-exporter counts are separate Python classes that each expose the same delivery counters; each getter reads its own class's fields."