diff --git a/Cargo.lock b/Cargo.lock index f443b6c3ea..b2597eaf8b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4456,6 +4456,7 @@ dependencies = [ "sqlbuild-python-base", "sqlbuild-python-compile", "sqlbuild-python-front", + "sqlbuild-python-output", "sqlbuild-python-planner", "sqlbuild-python-project", "sqlbuild-recall", @@ -4550,6 +4551,26 @@ dependencies = [ "uuid", ] +[[package]] +name = "sqlbuild-python-output" +version = "0.0.0" +dependencies = [ + "pyo3", + "serde_json", + "sqlbuild-analysis", + "sqlbuild-cache", + "sqlbuild-cli", + "sqlbuild-config", + "sqlbuild-core", + "sqlbuild-host", + "sqlbuild-orchestration", + "sqlbuild-python-base", + "sqlbuild-python-front", + "sqlbuild-python-planner", + "sqlbuild-python-project", + "sqlbuild-recall", +] + [[package]] name = "sqlbuild-python-planner" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 6386ddc847..9a8fc2b6b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ members = [ "crates/sqlbuild-python-project", "crates/sqlbuild-python-planner", "crates/sqlbuild-python-compile", + "crates/sqlbuild-python-output", "crates/sqlbuild-python", ] resolver = "2" @@ -65,9 +66,10 @@ order = [ "sqlbuild-python-project", "sqlbuild-python-planner", "sqlbuild-python-compile", + "sqlbuild-python-output", "sqlbuild-python", ] -python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python-planner", "sqlbuild-python-compile", "sqlbuild-python"] +python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python-planner", "sqlbuild-python-compile", "sqlbuild-python-output", "sqlbuild-python"] polyglot-floor = "sqlbuild-analysis" [workspace.package] @@ -192,6 +194,7 @@ sqlbuild-python-front = { path = "crates/sqlbuild-python-front" } sqlbuild-python-project = { path = "crates/sqlbuild-python-project" } sqlbuild-python-planner = { path = "crates/sqlbuild-python-planner" } sqlbuild-python-compile = { path = "crates/sqlbuild-python-compile" } +sqlbuild-python-output = { path = "crates/sqlbuild-python-output" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } sqlbuild-recall = { path = "crates/sqlbuild-recall" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } diff --git a/crates/README.md b/crates/README.md index a307c90e7e..d30136a3bd 100644 --- a/crates/README.md +++ b/crates/README.md @@ -33,14 +33,15 @@ tested and benchmarked with `cargo test` alone. | `sqlbuild-python-project` | The compiled project as PyO3 bindings, on top of `sqlbuild-python-front`: the project host and the Python views of compiled models, inputs and resources, the compiled-project snapshot, analysis classes and the analysis session, the project graph, references, lineage, fingerprints, contracts and semantic shapes, audit rendering, and the planner's identities, scope, selection, phases, fixture strategy and SQL test planning. | | `sqlbuild-python-planner` | The plan layer as PyO3 bindings, on top of `sqlbuild-python-project`: execution plans and their entries, planner phases, change detection, migrations, reference and cursor resolution, scenarios, strategy and warehouse inspection, plus freshness, consumption, microbatches, observations, lifecycle events and plan output. | | `sqlbuild-python-compile` | The compile layer as PyO3 bindings, on top of `sqlbuild-python-project` and beside `sqlbuild-python-planner`: the compile command and driver, input loops and compile-input attachment, project assembly, analysis and the analysis session, Rules, lineage, semantic checks, compile deltas and outputs, the DAG artifact, manifest, frontier, offline scope facts, SQL test glue and auditing. | -| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-compile`, `sqlbuild-python-planner`, `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. | +| `sqlbuild-python-output` | The output layer as PyO3 bindings, on top of `sqlbuild-python-planner` and beside `sqlbuild-python-compile`: build progress reporting, build result output and its documents, output capture and invocation records, cost telemetry, compute log storage and the Recall session and its stores. | +| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-output`, `sqlbuild-python-compile`, `sqlbuild-python-planner`, `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. | ## Layering Dependencies point one way, from the top of this graph to the bottom: ```text -sqlbuild-python -> sqlbuild-python-compile -> sqlbuild-python-planner -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base +sqlbuild-python -> sqlbuild-python-output -> sqlbuild-python-compile -> sqlbuild-python-planner -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base -> sqlbuild-project -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core diff --git a/crates/sqlbuild-python-output/Cargo.toml b/crates/sqlbuild-python-output/Cargo.toml new file mode 100644 index 0000000000..7b150daa06 --- /dev/null +++ b/crates/sqlbuild-python-output/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "sqlbuild-python-output" +description = "The output layer of the sqlbuild._native bindings: build progress, result output, output capture, cost telemetry, compute logs and the Recall session." +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +pyo3.workspace = true +serde_json.workspace = true +sqlbuild-analysis.workspace = true +sqlbuild-cache.workspace = true +sqlbuild-cli.workspace = true +sqlbuild-config.workspace = true +sqlbuild-core.workspace = true +sqlbuild-host.workspace = true +sqlbuild-orchestration.workspace = true +sqlbuild-python-base.workspace = true +sqlbuild-python-front.workspace = true +sqlbuild-python-project.workspace = true +sqlbuild-python-planner.workspace = true +sqlbuild-recall.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-python-output/src/build_progress/_helpers/audit_progress.rs b/crates/sqlbuild-python-output/src/build_progress/_helpers/audit_progress.rs new file mode 100644 index 0000000000..b35e50ddd0 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/_helpers/audit_progress.rs @@ -0,0 +1,38 @@ +//! Aggregate live state and plan-ordered result rows of standalone concurrent audits. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, Py, PyAny, PyResult}; +use sqlbuild_core::nested_progress::main::audit_resource_id::audit_resource_id; + +use crate::build_progress::_helpers::result_text::optional; +use crate::build_progress::_helpers::result_text::text; + +/// The identity an audit plan entry or result reports to the progress projector. +pub(crate) fn resource_id(value: &Bound<'_, PyAny>, name_field: &str) -> PyResult { + let kind: Option = match optional(value, "attached_target_kind")? { + Some(kind) => Some(kind), + None => Some(text(value, "attachment_kind")?), + }; + Ok(audit_resource_id( + &text(value, name_field)?, + kind.as_deref().unwrap_or(""), + ( + optional(value, "attached_target_name")?.as_deref(), + optional(value, "attached_column_name")?.as_deref(), + ), + )) +} + +/// Mutable reporter state; only held briefly and never across Python calls. +#[derive(Debug, Default)] +pub(crate) struct ReporterState { + pub(crate) completed_by_index: HashMap>>, + pub(crate) next_index: usize, + pub(crate) running: usize, + pub(crate) completed: usize, + pub(crate) current_group: Option, + pub(crate) closed: bool, + pub(crate) cursor_hidden: bool, + pub(crate) on_result: Option>, +} diff --git a/crates/sqlbuild-python-output/src/build_progress/_helpers/build_callbacks.rs b/crates/sqlbuild-python-output/src/build_progress/_helpers/build_callbacks.rs new file mode 100644 index 0000000000..e867be97e1 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/_helpers/build_callbacks.rs @@ -0,0 +1,131 @@ +//! Live progress rows, completion rows and runtime diagnostics of `sqb build`. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::build_progress::models::{SchedulerStateText, StatementText, TestResultText}; + +use crate::build_progress::_helpers::result_text::optional; +use crate::build_progress::_helpers::result_text::text; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +pub(crate) const RUNTIME_LOGGER: &str = "sqlbuild.executor.runtime"; +pub(crate) const PASS_OUTCOME: &str = "pass"; +pub(crate) const SCHEDULER_DIAGNOSTIC_LIMIT: usize = 50; +pub(crate) const QUERY_DIAGNOSTIC_LIMIT: usize = 25; + +/// Which executor result type a completion reports. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum ResultKind { + Model, + Seed, + Function, + Load, + PythonNode, + SqlTest, + Other, +} + +pub(crate) fn result_kind(result: &Bound<'_, PyAny>) -> PyResult { + for (field, kind) in [ + ("model_name", ResultKind::Model), + ("seed_name", ResultKind::Seed), + ("function_name", ResultKind::Function), + ("loader_name", ResultKind::Load), + ("node_name", ResultKind::PythonNode), + ("test_name", ResultKind::SqlTest), + ] { + if result.hasattr(field)? { + return Ok(kind); + } + } + Ok(ResultKind::Other) +} + +fn sql_test_resource_id(result: &Bound<'_, PyAny>) -> PyResult { + let name: String = text(result, "test_name")?; + let path: Bound<'_, PyAny> = native_field(result, "source_path")?; + let block: Bound<'_, PyAny> = native_field(result, "block_index")?; + let case: Bound<'_, PyAny> = native_field(result, "case_name")?; + if path.is_none() || block.is_none() || case.is_none() { + return Ok(format!("sql_test:{name}")); + } + Ok(format!( + "sql_test:{}:{}:{}", + path.call_method0("as_posix")?.str()?, + block.str()?, + case.str()? + )) +} + +/// The resource name and projector identity of a completion. +pub(crate) fn resource_identity(result: &Bound<'_, PyAny>) -> PyResult> { + Ok(match result_kind(result)? { + ResultKind::Model => { + let name: String = text(result, "model_name")?; + Some((name.clone(), format!("model:{name}"))) + } + ResultKind::Seed => { + let name: String = text(result, "seed_name")?; + Some((name.clone(), format!("seed:{name}"))) + } + ResultKind::Function => { + let name: String = text(result, "function_name")?; + let kind: String = text(result, "function_kind")?; + Some((name.clone(), format!("{kind}:{name}"))) + } + ResultKind::Load => { + let name: String = text(result, "source_name")?; + Some((name.clone(), format!("source:{name}"))) + } + ResultKind::PythonNode => { + let name: String = text(result, "node_name")?; + let kind: String = text(result, "kind")?; + Some((name.clone(), format!("{kind}:{name}"))) + } + ResultKind::SqlTest => Some((text(result, "test_name")?, sql_test_resource_id(result)?)), + ResultKind::Other => None, + }) +} + +/// Mutable callback state; only held briefly and never across Python calls. +#[derive(Debug, Default)] +pub(crate) struct CallbackState { + pub(crate) counter: usize, + pub(crate) node_name: String, + pub(crate) node_kind: String, + pub(crate) sub_message: String, + pub(crate) frame: usize, + pub(crate) line_active: bool, + pub(crate) cursor_hidden: bool, + pub(crate) stop_event: Option>, + pub(crate) thread: Option>, + pub(crate) tests_by_model: HashMap>, + pub(crate) scheduler_count: usize, + pub(crate) scheduler_omitted: usize, + pub(crate) last_scheduler: Option, + pub(crate) query_counts: HashMap, + pub(crate) query_omitted: usize, + pub(crate) last_query: Option, +} + +pub(crate) fn scheduler_state(state: &Bound<'_, PyAny>) -> PyResult { + Ok(SchedulerStateText { + running: native_field(state, "running")?.extract()?, + ready: native_field(state, "ready")?.extract()?, + waiting: native_field(state, "waiting")?.extract()?, + limit: native_field(state, "limit")?.extract()?, + aborted: native_field(state, "aborted")?.extract()?, + }) +} + +pub(crate) fn statement(telemetry: &Bound<'_, PyAny>) -> PyResult { + Ok(StatementText { + query_id: optional(telemetry, "query_id")?, + status: text(telemetry, "status")?, + elapsed_seconds: native_field(telemetry, "elapsed_seconds")?.extract()?, + resource_type: text(telemetry, "resource_type")?, + resource_name: text(telemetry, "resource_name")?, + phase: text(telemetry, "phase")?, + }) +} diff --git a/crates/sqlbuild-python-output/src/build_progress/_helpers/footer.rs b/crates/sqlbuild-python-output/src/build_progress/_helpers/footer.rs new file mode 100644 index 0000000000..b357c2c1c0 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/_helpers/footer.rs @@ -0,0 +1,24 @@ +//! The `sqb build` completion footer. + +use crate::build_progress::main::format_build_footer_with::format_build_footer_with; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; + +use crate::build_progress::_helpers::result_text::build_text; + +/// Render the build completion line, warnings, failures and skipped rollup. +#[pyfunction] +#[pyo3(signature = (*, result, elapsed, use_color, python_node_results = None))] +pub(crate) fn format_build_footer( + result: &Bound<'_, PyAny>, + elapsed: f64, + use_color: bool, + python_node_results: Option<&Bound<'_, PyAny>>, +) -> PyResult { + format_build_footer_with( + &build_text(result)?, + (elapsed, use_color), + python_node_results, + Vec::new(), + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/getters.rs b/crates/sqlbuild-python-output/src/build_progress/_helpers/getters.rs similarity index 62% rename from crates/sqlbuild-python/src/bindings/_helpers/build_progress/getters.rs rename to crates/sqlbuild-python-output/src/build_progress/_helpers/getters.rs index cc0e980ca6..a5595a15c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/getters.rs +++ b/crates/sqlbuild-python-output/src/build_progress/_helpers/getters.rs @@ -3,24 +3,10 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::build_progress::build_callbacks::BuildProgressCallbacks; +use crate::build_progress::models::BuildProgressCallbacks; use sqlbuild_python_base::build_progress::models::TransientLineCoordinator; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[ - GetterEntry { - class: class_of::, - read: read_build_progress_callbacks, - }, - GetterEntry { - class: class_of::, - read: read_transient_line_coordinator, - }, -]; - -fn read_build_progress_callbacks<'py>( +pub(crate) fn read_build_progress_callbacks<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -37,7 +23,7 @@ fn read_build_progress_callbacks<'py>( }) } -fn read_transient_line_coordinator<'py>( +pub(crate) fn read_transient_line_coordinator<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-output/src/build_progress/_helpers/mod.rs b/crates/sqlbuild-python-output/src/build_progress/_helpers/mod.rs new file mode 100644 index 0000000000..7208cda47c --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/_helpers/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod audit_progress; +pub(crate) mod build_callbacks; +pub(crate) mod footer; +pub(crate) mod getters; +pub(crate) mod nested_progress; +pub(crate) mod plan_facts; +pub(crate) mod progress_reporters; +pub(crate) mod progress_text; +pub(crate) mod result_text; diff --git a/crates/sqlbuild-python-output/src/build_progress/_helpers/nested_progress.rs b/crates/sqlbuild-python-output/src/build_progress/_helpers/nested_progress.rs new file mode 100644 index 0000000000..ef7bf8e6b2 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/_helpers/nested_progress.rs @@ -0,0 +1,65 @@ +//! One live spinner row for all running items and grouped completed rows of `sqb test`. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::nested_progress::models::NestedChildRow; + +use crate::build_progress::_helpers::result_text::text; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +pub(crate) const MIN_NAME_WIDTH: usize = 50; +pub(crate) const ITEM_OPTIONS: (&[&str], &[&str]) = ( + &["group_name", "item_name"], + &[ + "status_text", + "detail", + "error_code", + "error_help", + "error_message", + "child_rows", + "canonical_resource_name", + "canonical_resource_id", + ], +); + +/// Mutable reporter state; only held briefly and never across Python calls. +#[derive(Default)] +pub(crate) struct ReporterState { + pub(crate) printed_group: Option, + pub(crate) running: Vec<(String, String)>, + pub(crate) frame: usize, + pub(crate) line_active: bool, + pub(crate) cursor_hidden: bool, + pub(crate) stop_event: Option>, + pub(crate) thread: Option>, +} + +pub(crate) fn optional_argument( + arguments: &KeywordArguments<'_>, + name: &str, +) -> PyResult> { + if !arguments.passed(name)? { + return Ok(None); + } + let value: Bound<'_, PyAny> = arguments.value(name)?; + if value.is_none() { + return Ok(None); + } + value.extract().map(Some) +} + +pub(crate) fn child_rows(arguments: &KeywordArguments<'_>) -> PyResult> { + if !arguments.passed("child_rows")? { + return Ok(Vec::new()); + } + let mut rows: Vec = Vec::new(); + for row in arguments.value("child_rows")?.try_iter()? { + let row: Bound<'_, PyAny> = row?; + rows.push(NestedChildRow { + label: text(&row, "label")?, + name: text(&row, "name")?, + status_text: text(&row, "status_text")?, + detail: text(&row, "detail")?, + }); + } + Ok(rows) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs b/crates/sqlbuild-python-output/src/build_progress/_helpers/plan_facts.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs rename to crates/sqlbuild-python-output/src/build_progress/_helpers/plan_facts.rs index 915ef8e1f4..e630631e0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs +++ b/crates/sqlbuild-python-output/src/build_progress/_helpers/plan_facts.rs @@ -9,7 +9,7 @@ use sqlbuild_core::plan_display::main::model_execution_annotation::model_executi use sqlbuild_core::plan_display::main::model_resource_type::model_resource_type; use sqlbuild_core::plan_display::models::ModelLabelFacts; -use crate::bindings::_helpers::build_progress::result_text::text; +use crate::build_progress::_helpers::result_text::text; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_planner::plan_output::main::optional_facts::optional_facts; @@ -17,6 +17,7 @@ const SOURCE_TYPE: &str = "source"; const SQL_TEST_TYPE: &str = "sql_test"; /// What a build run's progress rows read from its plan. +#[derive(Debug)] pub(crate) struct BuildPlanFacts { pub(crate) total: usize, /// Display positions of every progress row, for the progress projector. diff --git a/crates/sqlbuild-python-output/src/build_progress/_helpers/progress_reporters.rs b/crates/sqlbuild-python-output/src/build_progress/_helpers/progress_reporters.rs new file mode 100644 index 0000000000..707d2db94a --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/_helpers/progress_reporters.rs @@ -0,0 +1,11 @@ +//! Warehouse connection and planning phase progress reporters. + +pub(crate) const CONNECTION_OPTIONS: (&[&str], &[&str]) = ( + &["adapter_name", "stream"], + &[ + "blank_line_before_start", + "blank_line_after_complete", + "use_color", + ], +); +pub(crate) const PROJECT_COMPILE_OPERATION: &str = "project_compile"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs b/crates/sqlbuild-python-output/src/build_progress/_helpers/progress_text.rs similarity index 65% rename from crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs rename to crates/sqlbuild-python-output/src/build_progress/_helpers/progress_text.rs index 15422b3ba7..65c9635636 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs +++ b/crates/sqlbuild-python-output/src/build_progress/_helpers/progress_text.rs @@ -3,69 +3,23 @@ use std::sync::{Mutex, MutexGuard}; use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyDict; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; -use sqlbuild_core::cli_text::models::CliStyle; -use sqlbuild_core::progress::main::execution_header_text::execution_header_text; -use sqlbuild_core::progress::main::format_execution_header::format_execution_header as format_header; -use sqlbuild_core::progress::main::format_expectation_detail::format_expectation_detail as format_detail; -use sqlbuild_core::progress::main::format_expectation_name::format_expectation_name as format_name; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::{pyclass, pymethods}; use sqlbuild_core::progress::main::format_inspection_record::format_inspection_record; use sqlbuild_core::progress::models::{ExpectationStep, InspectionRecord}; use sqlbuild_core::progress::types::DifferenceSample; use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_python_base::boundary::main::within::within; -use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::build_progress::main::progress::write_persistent::write_persistent; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; -const HEADER_OPTIONS: (&[&str], &[&str]) = ( +pub(crate) const HEADER_OPTIONS: (&[&str], &[&str]) = ( &["stream", "command", "target", "concurrency", "use_color"], &[], ); -/// Format command and execution context for a progress header. -#[pyfunction] -#[pyo3(signature = (*, command, target, concurrency))] -pub(crate) fn format_execution_header( - command: &str, - target: Option<&str>, - concurrency: i64, -) -> String { - format_header(command, target, concurrency) -} - -/// Write an execution header for command progress output. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn write_execution_header( - py: Python<'_>, - options: Option>, -) -> PyResult<()> { - let arguments: KeywordArguments<'_> = - KeywordArguments::read(py, "write_execution_header", options, HEADER_OPTIONS)?; - let command: String = arguments.value("command")?.extract()?; - let target: Option = arguments.value("target")?.extract()?; - let concurrency: i64 = arguments.value("concurrency")?.extract()?; - let style: CliStyle = CliStyle { - use_color: arguments.value("use_color")?.extract()?, - }; - let text: String = execution_header_text(style, (&command, target.as_deref(), concurrency)); - let stream: Bound<'_, PyAny> = arguments.value("stream")?; - stream.call_method1("write", (text,))?; - stream.call_method0("flush")?; - Ok(()) -} - -/// Format a comparison result name as a user-facing expectation name. -#[pyfunction] -pub(crate) fn format_expectation_name(model_name: &str) -> String { - format_name(model_name) -} - fn difference_samples(samples: &Bound<'_, PyAny>) -> PyResult> { let mut rows: Vec = Vec::new(); for sample in samples.try_iter()? { @@ -98,12 +52,6 @@ pub(crate) fn expectation_step(step: &Bound<'_, PyAny>) -> PyResult) -> PyResult { - Ok(format_detail(&expectation_step(step_result)?)) -} - fn inspection_record(record: &Bound<'_, PyAny>) -> PyResult { Ok(InspectionRecord { sql: native_field(record, "sql")?.extract()?, @@ -189,11 +137,3 @@ pub(crate) fn with_verbose_inspection<'py, T>( writer.get().write_total(py)?; Ok(value) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(format_execution_header, module)?)?; - module.add_function(wrap_pyfunction!(write_execution_header, module)?)?; - module.add_function(wrap_pyfunction!(format_expectation_name, module)?)?; - module.add_function(wrap_pyfunction!(format_expectation_detail, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs b/crates/sqlbuild-python-output/src/build_progress/_helpers/result_text.rs similarity index 82% rename from crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs rename to crates/sqlbuild-python-output/src/build_progress/_helpers/result_text.rs index f5b61e58f4..e99382c092 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs +++ b/crates/sqlbuild-python-output/src/build_progress/_helpers/result_text.rs @@ -1,5 +1,7 @@ //! Executor result objects read into the plain data build progress output renders. +use crate::build_progress::main::build_result_text::build_result_text; +use crate::build_progress::main::test_result_label::test_result_label; use std::collections::HashMap; use pyo3::IntoPyObjectExt; @@ -18,7 +20,7 @@ use sqlbuild_core::progress::models::ExpectationStep; use sqlbuild_core::test_command::main::parameterized_test_label::parameterized_test_label; use sqlbuild_core::test_command::models::TestParameter; use sqlbuild_host::models::{ - AuditEvaluationMode, AuditExecutionResult, BuildExecutionResult, BuildStatus, ExecutionStatus, + AuditEvaluationMode, AuditExecutionResult, BuildExecutionResult, ExecutionStatus, FunctionExecutionResult, HookExecutionResult, HookPhase, LifeCycleEvent, LifeCycleEventKind, LoadExecutionResult, MeasuredValue, ModelExecutionResult, SeedExecutionResult, SqlTestExecutionResult, StepResult, @@ -95,7 +97,7 @@ fn audit_result_text(audit: &AuditExecutionResult>) -> AuditRes } } -fn audits(value: &[AuditExecutionResult>]) -> Vec { +pub(crate) fn audits(value: &[AuditExecutionResult>]) -> Vec { value.iter().map(audit_result_text).collect() } @@ -136,7 +138,7 @@ pub(crate) fn model_text(model: &Bound<'_, PyAny>) -> PyResult model_result_text(model.py(), &ModelExecutionResult::read(model)?) } -fn model_result_text( +pub(crate) fn model_result_text( py: Python<'_>, model: &ModelExecutionResult>, ) -> PyResult { @@ -171,7 +173,7 @@ pub(crate) fn seed_text(seed: &Bound<'_, PyAny>) -> PyResult { Ok(seed_result_text(&SeedExecutionResult::read(seed)?)) } -fn seed_result_text(seed: &SeedExecutionResult) -> NodeResultText { +pub(crate) fn seed_result_text(seed: &SeedExecutionResult) -> NodeResultText { node_result_text( (&seed.seed_name, "seed"), (seed.status, seed.duration_ms), @@ -186,7 +188,7 @@ pub(crate) fn function_text(function: &Bound<'_, PyAny>) -> PyResult NodeResultText { +pub(crate) fn function_result_text(function: &FunctionExecutionResult) -> NodeResultText { node_result_text( (&function.function_name, &function.function_kind), (function.status, function.duration_ms), @@ -219,7 +221,7 @@ pub(crate) fn load_text(load: &Bound<'_, PyAny>) -> PyResult { Ok(load_result_text(&LoadExecutionResult::read(load)?)) } -fn load_result_text(load: &LoadExecutionResult>) -> LoadRowText { +pub(crate) fn load_result_text(load: &LoadExecutionResult>) -> LoadRowText { LoadRowText { source_name: load.source_name.clone(), loader_name: load.loader_name.clone(), @@ -278,7 +280,9 @@ fn test_step_text(step: &StepResult) -> TestStepText { } } -fn test_result_text(test: &SqlTestExecutionResult>) -> PyResult { +pub(crate) fn test_result_text( + test: &SqlTestExecutionResult>, +) -> PyResult { Ok(TestResultText { label: test_result_label(test)?, outcome: test.outcome.value().to_owned(), @@ -287,24 +291,6 @@ fn test_result_text(test: &SqlTestExecutionResult>) -> PyResult }) } -/// The shared progress/footer/command label of an already-converted executor test result. -pub(crate) fn test_result_label( - test: &SqlTestExecutionResult>, -) -> PyResult { - test_label_values( - &test.test_name, - test.source_path - .as_ref() - .map(|path| { - path.to_string_lossy() - .replace(std::path::MAIN_SEPARATOR, "/") - }) - .as_deref(), - &test.parameter_schema, - &test.parameter_values, - ) -} - pub(crate) fn python_node_text(node: &Bound<'_, PyAny>) -> PyResult { Ok(PythonNodeText { name: text(node, "node_name")?, @@ -321,50 +307,7 @@ pub(crate) fn build_text(result: &Bound<'_, PyAny>) -> PyResult build_result_text(py, &native) } -pub(crate) fn build_result_text( - py: Python<'_>, - native: &BuildExecutionResult>, -) -> PyResult { - Ok(BuildResultText { - failed: native.status == BuildStatus::Failed, - warning_count: native.warning_count, - insufficient_count: native.insufficient_count, - models: native - .model_results - .iter() - .map(|model| model_result_text(py, model)) - .collect::>()?, - seeds: native.seed_results.iter().map(seed_result_text).collect(), - functions: native - .function_results - .iter() - .map(function_result_text) - .collect(), - loads: native.load_results.iter().map(load_result_text).collect(), - tests: native - .test_results - .iter() - .map(test_result_text) - .collect::>()?, - source_audits: audits(&native.source_audit_results), - end_audits: audits(&native.end_audit_results), - }) -} - -/// The label of one test or result: its name, plus source and values for parameterized cases. -pub(crate) fn test_label(item: &Bound<'_, PyAny>, name_field: &str) -> PyResult { - let name: String = native_field(item, name_field)?.extract()?; - let source_path: Bound<'_, PyAny> = item.getattr("source_path")?; - let values = Vec::<(String, Bound<'_, PyAny>)>::read(&native_field(item, "parameter_values")?)?; - if source_path.is_none() || values.is_empty() { - return Ok(name); - } - let path: String = source_path.call_method0("as_posix")?.extract()?; - let declarations = Vec::>::read(&item.getattr("parameter_schema")?)?; - test_label_values(&name, Some(&path), &declarations, &values) -} - -fn test_label_values( +pub(crate) fn test_label_values( name: &str, path: Option<&str>, declarations: &[Bound<'_, PyAny>], diff --git a/crates/sqlbuild-python-output/src/build_progress/constants.rs b/crates/sqlbuild-python-output/src/build_progress/constants.rs new file mode 100644 index 0000000000..b4fbf5d4d5 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/constants.rs @@ -0,0 +1,20 @@ +//! Public constants of build progress. + +use crate::build_progress::_helpers::getters::read_build_progress_callbacks; +use crate::build_progress::_helpers::getters::read_transient_line_coordinator; +use crate::build_progress::models::BuildProgressCallbacks; +use sqlbuild_python_base::build_progress::models::TransientLineCoordinator; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_build_progress_callbacks, + }, + GetterEntry { + class: class_of::, + read: read_transient_line_coordinator, + }, +]; diff --git a/crates/sqlbuild-python-output/src/build_progress/main/batch_complete_callback.rs b/crates/sqlbuild-python-output/src/build_progress/main/batch_complete_callback.rs new file mode 100644 index 0000000000..2ed7aef08c --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/batch_complete_callback.rs @@ -0,0 +1,21 @@ +//! A callable that forwards each completed microbatch to `callbacks`, built without a Python lookup. + +use crate::build_progress::models::BuildProgressCallbacks; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyCFunction, PyDict, PyTuple, PyTupleMethods}; + +/// A callable that forwards each completed microbatch to `callbacks`, built without a Python lookup. +pub fn batch_complete_callback<'py>(callbacks: &Bound<'py, PyAny>) -> PyResult> { + let owned: Py = callbacks + .downcast::()? + .clone() + .unbind(); + let forward = move |arguments: &Bound<'_, PyTuple>, + _keywords: Option<&Bound<'_, PyDict>>| + -> PyResult<()> { + owned + .get() + .write_batch_complete(arguments.py(), &arguments.get_item(0)?) + }; + Ok(PyCFunction::new_closure(callbacks.py(), None, None, forward)?.into_any()) +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/build_result_text.rs b/crates/sqlbuild-python-output/src/build_progress/main/build_result_text.rs new file mode 100644 index 0000000000..c6a0ae240c --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/build_result_text.rs @@ -0,0 +1,42 @@ +//! Build result text. + +use crate::build_progress::_helpers::result_text::audits; +use crate::build_progress::_helpers::result_text::function_result_text; +use crate::build_progress::_helpers::result_text::load_result_text; +use crate::build_progress::_helpers::result_text::model_result_text; +use crate::build_progress::_helpers::result_text::seed_result_text; +use crate::build_progress::_helpers::result_text::test_result_text; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_core::build_progress::models::BuildResultText; +use sqlbuild_host::models::{BuildExecutionResult, BuildStatus}; + +/// Build result text. +pub fn build_result_text( + py: Python<'_>, + native: &BuildExecutionResult>, +) -> PyResult { + Ok(BuildResultText { + failed: native.status == BuildStatus::Failed, + warning_count: native.warning_count, + insufficient_count: native.insufficient_count, + models: native + .model_results + .iter() + .map(|model| model_result_text(py, model)) + .collect::>()?, + seeds: native.seed_results.iter().map(seed_result_text).collect(), + functions: native + .function_results + .iter() + .map(function_result_text) + .collect(), + loads: native.load_results.iter().map(load_result_text).collect(), + tests: native + .test_results + .iter() + .map(test_result_text) + .collect::>()?, + source_audits: audits(&native.source_audit_results), + end_audits: audits(&native.end_audit_results), + }) +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/check_callback.rs b/crates/sqlbuild-python-output/src/build_progress/main/check_callback.rs new file mode 100644 index 0000000000..22b078cd90 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/check_callback.rs @@ -0,0 +1,27 @@ +//! Forward a Python check lifecycle callback directly to its native progress owner. + +use crate::build_progress::models::BuildProgressCallbacks; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyCFunction, PyDict, PyTuple, PyTupleMethods}; + +/// Forward a Python check lifecycle callback directly to its native progress owner. +pub fn check_callback<'py>( + callbacks: &Bound<'py, PyAny>, + completion: bool, +) -> PyResult> { + let owned: Py = callbacks + .downcast::()? + .clone() + .unbind(); + let forward = move |arguments: &Bound<'_, PyTuple>, + _keywords: Option<&Bound<'_, PyDict>>| + -> PyResult<()> { + let value = arguments.get_item(0)?; + if completion { + owned.get().on_check_complete(arguments.py(), &value) + } else { + owned.get().on_check_start(arguments.py(), value.extract()?) + } + }; + Ok(PyCFunction::new_closure(callbacks.py(), None, None, forward)?.into_any()) +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/format_build_footer_with.rs b/crates/sqlbuild-python-output/src/build_progress/main/format_build_footer_with.rs new file mode 100644 index 0000000000..adbb9a1ae0 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/format_build_footer_with.rs @@ -0,0 +1,24 @@ +//! The build footer, counting `checks` (Python check results) with the Python node results. + +use crate::build_progress::_helpers::result_text::python_node_text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_core::build_progress::main::format_build_footer::format_build_footer as footer; +use sqlbuild_core::build_progress::models::{BuildResultText, PythonNodeText}; +use sqlbuild_core::cli_text::models::CliStyle; + +/// The build footer, counting `checks` (Python check results) with the Python node results. +pub fn format_build_footer_with( + result: &BuildResultText, + (elapsed, use_color): (f64, bool), + python_node_results: Option<&Bound<'_, PyAny>>, + checks: Vec, +) -> PyResult { + let mut python: Vec = Vec::new(); + if let Some(results) = python_node_results { + for node in results.try_iter()? { + python.push(python_node_text(&node?)?); + } + } + python.extend(checks); + Ok(footer(result, &python, elapsed, CliStyle { use_color })) +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/format_execution_header.rs b/crates/sqlbuild-python-output/src/build_progress/main/format_execution_header.rs new file mode 100644 index 0000000000..8a288aebb8 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/format_execution_header.rs @@ -0,0 +1,11 @@ +//! Format command and execution context for a progress header. + +use pyo3::pyfunction; +use sqlbuild_core::progress::main::format_execution_header::format_execution_header as format_header; + +/// Format command and execution context for a progress header. +#[pyfunction] +#[pyo3(signature = (*, command, target, concurrency))] +pub fn format_execution_header(command: &str, target: Option<&str>, concurrency: i64) -> String { + format_header(command, target, concurrency) +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/format_expectation_detail.rs b/crates/sqlbuild-python-output/src/build_progress/main/format_expectation_detail.rs new file mode 100644 index 0000000000..16650afc38 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/format_expectation_detail.rs @@ -0,0 +1,12 @@ +//! Format failing row count detail for an expectation row. + +use crate::build_progress::_helpers::progress_text::expectation_step; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::pyfunction; +use sqlbuild_core::progress::main::format_expectation_detail::format_expectation_detail as format_detail; + +/// Format failing row count detail for an expectation row. +#[pyfunction] +pub fn format_expectation_detail(step_result: &Bound<'_, PyAny>) -> PyResult { + Ok(format_detail(&expectation_step(step_result)?)) +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/format_expectation_name.rs b/crates/sqlbuild-python-output/src/build_progress/main/format_expectation_name.rs new file mode 100644 index 0000000000..a94a9be94d --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/format_expectation_name.rs @@ -0,0 +1,10 @@ +//! Format a comparison result name as a user-facing expectation name. + +use pyo3::pyfunction; +use sqlbuild_core::progress::main::format_expectation_name::format_expectation_name as format_name; + +/// Format a comparison result name as a user-facing expectation name. +#[pyfunction] +pub fn format_expectation_name(model_name: &str) -> String { + format_name(model_name) +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/mod.rs b/crates/sqlbuild-python-output/src/build_progress/main/mod.rs new file mode 100644 index 0000000000..d806b9d190 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/mod.rs @@ -0,0 +1,17 @@ +pub mod batch_complete_callback; +pub mod build_result_text; +pub mod check_callback; +pub mod format_build_footer_with; +pub mod format_execution_header; +pub mod format_expectation_detail; +pub mod format_expectation_name; +pub mod register_audit_progress; +pub mod register_build_callbacks; +pub mod register_footer; +pub mod register_nested_progress; +pub mod register_progress_reporters; +pub mod register_progress_text; +pub mod test_label; +pub mod test_result_label; +pub mod with_verbose_inspection; +pub mod write_execution_header; diff --git a/crates/sqlbuild-python-output/src/build_progress/main/register_audit_progress.rs b/crates/sqlbuild-python-output/src/build_progress/main/register_audit_progress.rs new file mode 100644 index 0000000000..46c9c704ae --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/register_audit_progress.rs @@ -0,0 +1,9 @@ +//! Register the audit progress functions and classes. + +use crate::build_progress::models::AuditProgressReporter; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the audit progress functions and classes. +pub fn register_audit_progress(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/register_build_callbacks.rs b/crates/sqlbuild-python-output/src/build_progress/main/register_build_callbacks.rs new file mode 100644 index 0000000000..5eadfc1041 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/register_build_callbacks.rs @@ -0,0 +1,9 @@ +//! Register the build callbacks functions and classes. + +use crate::build_progress::models::BuildProgressCallbacks; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the build callbacks functions and classes. +pub fn register_build_callbacks(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/register_footer.rs b/crates/sqlbuild-python-output/src/build_progress/main/register_footer.rs new file mode 100644 index 0000000000..485fda830f --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/register_footer.rs @@ -0,0 +1,10 @@ +//! Register the footer functions and classes. + +use crate::build_progress::_helpers::footer::format_build_footer; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the footer functions and classes. +pub fn register_footer(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_build_footer, module)?) +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/register_nested_progress.rs b/crates/sqlbuild-python-output/src/build_progress/main/register_nested_progress.rs new file mode 100644 index 0000000000..2cdd74dc87 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/register_nested_progress.rs @@ -0,0 +1,9 @@ +//! Register the nested progress functions and classes. + +use crate::build_progress::models::NestedCommandProgressCallbacks; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the nested progress functions and classes. +pub fn register_nested_progress(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/register_progress_reporters.rs b/crates/sqlbuild-python-output/src/build_progress/main/register_progress_reporters.rs new file mode 100644 index 0000000000..b42ade3287 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/register_progress_reporters.rs @@ -0,0 +1,13 @@ +//! Register the progress reporters functions and classes. + +use crate::build_progress::models::ConnectionProgressReporter; +use crate::build_progress::models::DbtInitProgressReporter; +use crate::build_progress::models::PlanningProgressReporter; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the progress reporters functions and classes. +pub fn register_progress_reporters(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::() +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/register_progress_text.rs b/crates/sqlbuild-python-output/src/build_progress/main/register_progress_text.rs new file mode 100644 index 0000000000..6544efb485 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/register_progress_text.rs @@ -0,0 +1,18 @@ +//! Register the progress text functions and classes. + +use crate::build_progress::_helpers::progress_text::VerboseInspectionWriter; +use crate::build_progress::main::format_execution_header::format_execution_header; +use crate::build_progress::main::format_expectation_detail::format_expectation_detail; +use crate::build_progress::main::format_expectation_name::format_expectation_name; +use crate::build_progress::main::write_execution_header::write_execution_header; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the progress text functions and classes. +pub fn register_progress_text(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(format_execution_header, module)?)?; + module.add_function(wrap_pyfunction!(write_execution_header, module)?)?; + module.add_function(wrap_pyfunction!(format_expectation_name, module)?)?; + module.add_function(wrap_pyfunction!(format_expectation_detail, module)?) +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/test_label.rs b/crates/sqlbuild-python-output/src/build_progress/main/test_label.rs new file mode 100644 index 0000000000..2b42977821 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/test_label.rs @@ -0,0 +1,19 @@ +//! The label of one test or result: its name, plus source and values for parameterized cases. + +use crate::build_progress::_helpers::result_text::test_label_values; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::native_host::types::ReadResult; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The label of one test or result: its name, plus source and values for parameterized cases. +pub fn test_label(item: &Bound<'_, PyAny>, name_field: &str) -> PyResult { + let name: String = native_field(item, name_field)?.extract()?; + let source_path: Bound<'_, PyAny> = item.getattr("source_path")?; + let values = Vec::<(String, Bound<'_, PyAny>)>::read(&native_field(item, "parameter_values")?)?; + if source_path.is_none() || values.is_empty() { + return Ok(name); + } + let path: String = source_path.call_method0("as_posix")?.extract()?; + let declarations = Vec::>::read(&item.getattr("parameter_schema")?)?; + test_label_values(&name, Some(&path), &declarations, &values) +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/test_result_label.rs b/crates/sqlbuild-python-output/src/build_progress/main/test_result_label.rs new file mode 100644 index 0000000000..07f8b786f5 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/test_result_label.rs @@ -0,0 +1,21 @@ +//! The shared progress/footer/command label of an already-converted executor test result. + +use crate::build_progress::_helpers::result_text::test_label_values; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_host::models::SqlTestExecutionResult; + +/// The shared progress/footer/command label of an already-converted executor test result. +pub fn test_result_label(test: &SqlTestExecutionResult>) -> PyResult { + test_label_values( + &test.test_name, + test.source_path + .as_ref() + .map(|path| { + path.to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/") + }) + .as_deref(), + &test.parameter_schema, + &test.parameter_values, + ) +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/with_verbose_inspection.rs b/crates/sqlbuild-python-output/src/build_progress/main/with_verbose_inspection.rs new file mode 100644 index 0000000000..0ab6cb2fd8 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/with_verbose_inspection.rs @@ -0,0 +1,15 @@ +//! Run `body`, writing each inspection read and then a total to stderr while `enabled`. + +use crate::build_progress::_helpers::progress_text; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; + +/// Run `body`, writing each inspection read and then a total to stderr while `enabled`. +#[inline] +pub fn with_verbose_inspection<'py, T>( + py: Python<'py>, + enabled: bool, + body: impl FnOnce() -> PyResult, +) -> PyResult { + progress_text::with_verbose_inspection(py, enabled, body) +} diff --git a/crates/sqlbuild-python-output/src/build_progress/main/write_execution_header.rs b/crates/sqlbuild-python-output/src/build_progress/main/write_execution_header.rs new file mode 100644 index 0000000000..2cf9450005 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/main/write_execution_header.rs @@ -0,0 +1,28 @@ +//! Write an execution header for command progress output. + +use crate::build_progress::_helpers::progress_text::HEADER_OPTIONS; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::progress::main::execution_header_text::execution_header_text; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// Write an execution header for command progress output. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn write_execution_header(py: Python<'_>, options: Option>) -> PyResult<()> { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "write_execution_header", options, HEADER_OPTIONS)?; + let command: String = arguments.value("command")?.extract()?; + let target: Option = arguments.value("target")?.extract()?; + let concurrency: i64 = arguments.value("concurrency")?.extract()?; + let style: CliStyle = CliStyle { + use_color: arguments.value("use_color")?.extract()?, + }; + let text: String = execution_header_text(style, (&command, target.as_deref(), concurrency)); + let stream: Bound<'_, PyAny> = arguments.value("stream")?; + stream.call_method1("write", (text,))?; + stream.call_method0("flush")?; + Ok(()) +} diff --git a/crates/sqlbuild-python-output/src/build_progress/mod.rs b/crates/sqlbuild-python-output/src/build_progress/mod.rs new file mode 100644 index 0000000000..53aa414f96 --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/mod.rs @@ -0,0 +1,6 @@ +//! Build progress reporting: progress reporters and callbacks, nested and audit progress, plan facts, result text and the footer. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-output/src/build_progress/models.rs b/crates/sqlbuild-python-output/src/build_progress/models.rs new file mode 100644 index 0000000000..c7140246bf --- /dev/null +++ b/crates/sqlbuild-python-output/src/build_progress/models.rs @@ -0,0 +1,1491 @@ +//! Public records of build progress. + +use crate::build_progress::_helpers::audit_progress::resource_id; +use crate::build_progress::_helpers::build_callbacks::CallbackState; +use crate::build_progress::_helpers::build_callbacks::PASS_OUTCOME; +use crate::build_progress::_helpers::build_callbacks::QUERY_DIAGNOSTIC_LIMIT; +use crate::build_progress::_helpers::build_callbacks::RUNTIME_LOGGER; +use crate::build_progress::_helpers::build_callbacks::ResultKind; +use crate::build_progress::_helpers::build_callbacks::SCHEDULER_DIAGNOSTIC_LIMIT; +use crate::build_progress::_helpers::build_callbacks::resource_identity; +use crate::build_progress::_helpers::build_callbacks::result_kind; +use crate::build_progress::_helpers::build_callbacks::scheduler_state; +use crate::build_progress::_helpers::build_callbacks::statement; +use crate::build_progress::_helpers::nested_progress::ITEM_OPTIONS; +use crate::build_progress::_helpers::nested_progress::MIN_NAME_WIDTH; +use crate::build_progress::_helpers::nested_progress::child_rows; +use crate::build_progress::_helpers::nested_progress::optional_argument; +use crate::build_progress::_helpers::plan_facts::BuildPlanFacts; +use crate::build_progress::_helpers::plan_facts::build_plan_facts; +use crate::build_progress::_helpers::progress_reporters::CONNECTION_OPTIONS; +use crate::build_progress::_helpers::progress_reporters::PROJECT_COMPILE_OPERATION; +use crate::build_progress::_helpers::result_text::audit_text; +use crate::build_progress::_helpers::result_text::function_text; +use crate::build_progress::_helpers::result_text::load_text; +use crate::build_progress::_helpers::result_text::model_text; +use crate::build_progress::_helpers::result_text::seed_text; +use crate::build_progress::_helpers::result_text::test_text; +use crate::build_progress::_helpers::result_text::text; +use crate::result_output::models::ExecutionEventWriter; +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use sqlbuild_core::build_progress::main::batch_progress_message::batch_progress_message; +use sqlbuild_core::build_progress::main::query_diagnostic_message::query_diagnostic_message; +use sqlbuild_core::build_progress::main::render_build_spinner_line::render_build_spinner_line; +use sqlbuild_core::build_progress::main::render_failed_test_result::render_failed_test_result; +use sqlbuild_core::build_progress::main::render_model_result::render_model_result; +use sqlbuild_core::build_progress::main::render_resource_result::render_resource_result; +use sqlbuild_core::build_progress::main::runtime_diagnostic_summary::runtime_diagnostic_summary; +use sqlbuild_core::build_progress::main::scheduler_diagnostic_message::scheduler_diagnostic_message; +use sqlbuild_core::build_progress::models::AuditResultText; +use sqlbuild_core::build_progress::models::BatchProgressText; +use sqlbuild_core::build_progress::models::ErrorText; +use sqlbuild_core::build_progress::models::ModelRowFacts; +use sqlbuild_core::build_progress::models::ProgressLayout; +use sqlbuild_core::build_progress::models::ResourceResult; +use sqlbuild_core::build_progress::models::SchedulerStateText; +use sqlbuild_core::build_progress::models::SpinnerRow; +use sqlbuild_core::build_progress::models::StatementText; +use sqlbuild_core::build_progress::models::TestResultText; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::nested_progress::main::audit_group_name::audit_group_name; +use sqlbuild_core::nested_progress::main::render_audit_aggregate::render_audit_aggregate; +use sqlbuild_core::nested_progress::main::render_audit_row::render_audit_row; +use sqlbuild_core::nested_progress::main::render_group_header::render_group_header; +use sqlbuild_core::nested_progress::main::render_nested_item::render_nested_item; +use sqlbuild_core::nested_progress::main::render_nested_spinner_line::render_nested_spinner_line; +use sqlbuild_core::nested_progress::models::AuditAggregate; +use sqlbuild_core::nested_progress::models::NestedItem; +use sqlbuild_core::nested_progress::models::NestedLayout; +use sqlbuild_core::progress::main::connection_phase_line::connection_phase_line; +use sqlbuild_core::progress::main::connection_start_message::connection_start_message; +use sqlbuild_core::progress::main::is_planning_completion_message::is_planning_completion_message; +use sqlbuild_core::progress::main::is_planning_failure_message::is_planning_failure_message; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::boundary::main::within::within; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_progress::constants::CLEAR_LINE; +use sqlbuild_python_base::build_progress::constants::HIDE_CURSOR; +use sqlbuild_python_base::build_progress::constants::SHOW_CURSOR; +use sqlbuild_python_base::build_progress::constants::SPINNER_FRAMES; +use sqlbuild_python_base::build_progress::constants::SPINNER_TICK_SECONDS; +use sqlbuild_python_base::build_progress::main::progress::current_projector::current_projector; +use sqlbuild_python_base::build_progress::main::progress::projector_operation_active::projector_operation_active; +use sqlbuild_python_base::build_progress::main::progress::write_persistent::write_persistent; +use sqlbuild_python_base::build_progress::main::terminal::claim::claim; +use sqlbuild_python_base::build_progress::main::terminal::consume_terminal::consume_terminal; +use sqlbuild_python_base::build_progress::main::terminal::flush::flush; +use sqlbuild_python_base::build_progress::main::terminal::join_spinner::join_spinner; +use sqlbuild_python_base::build_progress::main::terminal::projector::projector; +use sqlbuild_python_base::build_progress::main::terminal::release::release; +use sqlbuild_python_base::build_progress::main::terminal::shared_lines::shared_lines; +use sqlbuild_python_base::build_progress::main::terminal::start_spinner::start_spinner; +use sqlbuild_python_base::build_progress::main::terminal::terminal_columns::terminal_columns; +use sqlbuild_python_base::build_progress::main::terminal::tty_flag::tty_flag; +use sqlbuild_python_base::build_progress::main::terminal::write::write; +use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; +use sqlbuild_python_base::build_progress::models::NativeProgressProjector; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::MutexGuard; +use std::sync::PoisonError; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::time::Instant; + +/// Render aggregate live state and flush completed audit rows in plan order. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct AuditProgressReporter { + pub(crate) stream: Py, + pub(crate) lines: Py, + pub(crate) lock: Py, + pub(crate) style: CliStyle, + pub(crate) is_tty: bool, + pub(crate) projector: Option>, + pub(crate) indexes: HashMap, + pub(crate) total: usize, + pub(crate) worker_limit: i64, + pub(crate) state: Mutex, +} + +impl AuditProgressReporter { + pub(crate) fn state( + &self, + ) -> MutexGuard<'_, crate::build_progress::_helpers::audit_progress::ReporterState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + pub(crate) fn index_of(&self, identity: String) -> PyResult { + self.indexes + .get(&identity) + .copied() + .ok_or_else(|| PyKeyError::new_err(identity)) + } + + pub(crate) fn render_aggregate(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + let (closed, cursor_hidden, running, completed) = { + let state = reporter.state(); + ( + state.closed, + state.cursor_hidden, + state.running, + state.completed, + ) + }; + if !reporter.is_tty || closed { + return Ok(()); + } + let stream: &Bound<'_, PyAny> = reporter.stream.bind(py); + claim(reporter.lines.bind(py), stream, slf.as_any())?; + if !cursor_hidden { + write(stream, HIDE_CURSOR)?; + reporter.state().cursor_hidden = true; + } + write( + stream, + &render_audit_aggregate(AuditAggregate { + running, + completed, + total: reporter.total, + workers: reporter.worker_limit, + }), + )?; + flush(stream) + } + + pub(crate) fn flush_ready(&self, py: Python<'_>) -> PyResult<()> { + loop { + let ready: Option> = { + let mut state = self.state(); + let index: usize = state.next_index; + let Some(result) = state.completed_by_index.remove(&index) else { + return Ok(()); + }; + state.next_index += 1; + result + }; + if let Some(result) = ready { + self.write_result(py, result.bind(py))?; + } + } + } + + pub(crate) fn write_result(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + if self.is_tty { + write(stream, CLEAR_LINE)?; + } + let audit: AuditResultText = audit_text(result)?; + let group: String = audit_group_name(&audit); + let previous: Option = self.state().current_group.clone(); + if previous.as_ref() != Some(&group) { + write( + stream, + &render_group_header(&group, previous.is_none(), self.style), + )?; + self.state().current_group = Some(group); + } + let duration_ms: Option = match &self.projector { + Some(projector) => consume_terminal( + projector.bind(py), + &audit.audit_name, + Some(&resource_id(result, "audit_name")?), + )?, + None => None, + }; + write(stream, &render_audit_row(&audit, duration_ms, self.style))?; + flush(stream)?; + let callback: Option> = self + .state() + .on_result + .as_ref() + .map(|callback| callback.clone_ref(py)); + if let Some(callback) = callback { + callback.bind(py).call1((result,))?; + } + Ok(()) + } + + pub(crate) fn locked( + &self, + py: Python<'_>, + body: impl FnOnce() -> PyResult, + ) -> PyResult { + within(self.lock.bind(py), |_| body()) + } +} + +#[pymethods] +impl AuditProgressReporter { + #[new] + #[pyo3(signature = (**options))] + pub fn new(py: Python<'_>, options: Option>) -> PyResult { + let arguments = KeywordArguments::read( + py, + "AuditProgressReporter", + options, + (&["entries", "worker_limit", "stream", "use_color"], &[]), + )?; + let stream: Bound<'_, PyAny> = arguments.value("stream")?; + let lines: Bound<'_, PyAny> = shared_lines(py)?; + let mut indexes: HashMap = HashMap::new(); + let mut total: usize = 0; + for entry in arguments.value("entries")?.try_iter()? { + indexes.insert(resource_id(&entry?, "name")?, total); + total += 1; + } + Ok(Self { + is_tty: tty_flag(&stream)?, + stream: stream.unbind(), + lock: native_field(&lines, "lock")?.unbind(), + lines: lines.unbind(), + style: CliStyle { + use_color: arguments.value("use_color")?.is_truthy()?, + }, + projector: projector(py)?, + indexes, + total, + worker_limit: arguments.value("worker_limit")?.extract()?, + state: Mutex::new( + crate::build_progress::_helpers::audit_progress::ReporterState::default(), + ), + }) + } + + /// Install plan-ordered result enrichment before dispatch starts. + pub(crate) fn set_result_callback(&self, callback: Py) { + self.state().on_result = Some(callback); + } + + /// Record a worker start without mutating one-item spinner state. + pub(crate) fn on_item_start(slf: &Bound<'_, Self>, entry: &Bound<'_, PyAny>) -> PyResult<()> { + let _ = entry; + slf.get().locked(slf.py(), || { + slf.get().state().running += 1; + Self::render_aggregate(slf) + }) + } + + /// Flush every newly contiguous plan-order result row. + pub(crate) fn on_item_complete( + slf: &Bound<'_, Self>, + result: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + reporter.locked(py, || { + let index: usize = reporter.index_of(resource_id(result, "audit_name")?)?; + reporter + .state() + .completed_by_index + .insert(index, Some(result.clone().unbind())); + reporter.flush_ready(py)?; + Self::render_aggregate(slf) + }) + } + + /// Update aggregate state immediately without projecting a result row. + pub(crate) fn on_item_physical_complete( + slf: &Bound<'_, Self>, + result: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let _ = result; + slf.get().locked(slf.py(), || { + { + let mut state = slf.get().state(); + state.running = state.running.saturating_sub(1); + state.completed += 1; + } + Self::render_aggregate(slf) + }) + } + + /// Advance aggregate and ordering state for a fatal audit exception. + pub(crate) fn on_item_error(slf: &Bound<'_, Self>, entry: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + reporter.locked(py, || { + let index: usize = reporter.index_of(resource_id(entry, "name")?)?; + { + let mut state = reporter.state(); + state.running = state.running.saturating_sub(1); + state.completed += 1; + state.completed_by_index.insert(index, None); + } + reporter.flush_ready(py)?; + Self::render_aggregate(slf) + }) + } + + /// Restore the terminal after the final aggregate update. + pub(crate) fn close(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + reporter.locked(py, || { + reporter.state().closed = true; + release(reporter.lines.bind(py), slf.as_any())?; + if !reporter.is_tty { + return Ok(()); + } + let stream: &Bound<'_, PyAny> = reporter.stream.bind(py); + write(stream, CLEAR_LINE)?; + let hidden: bool = std::mem::take(&mut reporter.state().cursor_hidden); + if hidden { + write(stream, SHOW_CURSOR)?; + } + flush(stream) + }) + } + + /// Erase the aggregate row so a persistent line can take its place. + pub(crate) fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, CLEAR_LINE)?; + flush(stream) + }) + } + + /// Draw the aggregate row again below persistent output. + pub(crate) fn redraw_transient_line(slf: &Bound<'_, Self>) -> PyResult<()> { + slf.get().locked(slf.py(), || Self::render_aggregate(slf)) + } +} + +/// Encapsulates live build progress output state and callbacks. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct BuildProgressCallbacks { + pub worker: Option>, + pub(crate) plan: Py, + pub(crate) stream: Py, + pub(crate) lines: Py, + pub(crate) lock: Py, + pub(crate) event_writer: Py, + pub(crate) projector: Option>, + pub(crate) facts: BuildPlanFacts, + pub(crate) style: CliStyle, + pub(crate) verbose: bool, + pub(crate) debug: bool, + pub(crate) is_tty: bool, + pub(crate) started: Instant, + pub(crate) state: Mutex, +} + +impl BuildProgressCallbacks { + pub(crate) fn state(&self) -> MutexGuard<'_, CallbackState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + pub(crate) fn locked( + &self, + py: Python<'_>, + body: impl FnOnce() -> PyResult, + ) -> PyResult { + within(self.lock.bind(py), |_| body()) + } + + pub(crate) fn layout(&self, py: Python<'_>) -> PyResult { + Ok(ProgressLayout { + total: self.facts.total, + name_width: self.facts.name_width, + columns: terminal_columns(py)?, + verbose: self.verbose, + }) + } + + pub(crate) fn emit(&self, py: Python<'_>, text: &str) -> PyResult<()> { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, text)?; + flush(stream) + } + + pub(crate) fn write_spinner_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + let (row, frame) = { + let mut state = self.state(); + if !state.line_active { + return Ok(()); + } + let frame: usize = state.frame; + state.frame = (frame + 1) % SPINNER_FRAMES.len(); + ( + SpinnerRow { + position: state.counter + 1, + resource_kind: state.node_kind.clone(), + name: state.node_name.clone(), + message: state.sub_message.clone(), + }, + frame, + ) + }; + self.emit( + py, + &render_build_spinner_line(&row, frame, self.layout(py)?, self.style), + ) + }) + } + + pub(crate) fn set_cursor(&self, py: Python<'_>, hidden: bool) -> PyResult<()> { + if self.state().cursor_hidden == hidden { + return Ok(()); + } + self.locked(py, || { + self.emit(py, if hidden { HIDE_CURSOR } else { SHOW_CURSOR }) + })?; + self.state().cursor_hidden = hidden; + Ok(()) + } + + pub(crate) fn stop_spinner(&self, py: Python<'_>) -> PyResult<()> { + let (event, thread) = { + let mut state = self.state(); + (state.stop_event.take(), state.thread.take()) + }; + if let Some(event) = event { + event.bind(py).call_method0("set")?; + } + if let Some(thread) = thread { + join_spinner(thread.bind(py))?; + } + Ok(()) + } + + /// Stop the spinner and erase its row before a persistent row is written. + pub(crate) fn retire_live_row(slf: &Bound<'_, Self>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let callbacks: &Self = slf.get(); + callbacks.stop_spinner(py)?; + if !callbacks.is_tty { + return Ok(()); + } + callbacks.locked(py, || { + callbacks.state().line_active = false; + release(callbacks.lines.bind(py), slf.as_any())?; + callbacks.emit(py, CLEAR_LINE) + })?; + callbacks.set_cursor(py, false) + } + + pub(crate) fn write_runtime_diagnostic(&self, py: Python<'_>, message: &str) -> PyResult<()> { + imported(py, "logging", "getLogger")? + .call1((RUNTIME_LOGGER,))? + .call_method1("debug", (message,))?; + if self.debug { + return Ok(()); + } + self.write_persistent(py, message) + } + + /// Write one permanent line through the shared line coordinator. + pub(crate) fn write_persistent(&self, py: Python<'_>, message: &str) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("stream", self.stream.bind(py))?; + options.set_item("text", format!("{message}\n"))?; + self.lines + .bind(py) + .call_method("write_persistent", (), Some(&options))?; + Ok(()) + } + + /// Keep a line per completed microbatch when no live spinner row shows batch progress. + pub(crate) fn write_batch_complete( + &self, + py: Python<'_>, + progress: &Bound<'_, PyAny>, + ) -> PyResult<()> { + if self.is_tty { + return Ok(()); + } + let batch: BatchProgressText = BatchProgressText { + model_name: text(progress, "model_name")?, + batch_number: native_field(progress, "batch_number")?.extract()?, + batch_count: native_field(progress, "batch_count")?.extract()?, + window: text(progress, "window")?, + rows_affected: native_field(progress, "rows_affected")?.extract()?, + elapsed_seconds: native_field(progress, "elapsed_seconds")?.extract()?, + }; + self.write_persistent(py, &batch_progress_message(&batch)) + } + + pub(crate) fn write_node_result( + slf: &Bound<'_, Self>, + result: &Bound<'_, PyAny>, + kind: ResultKind, + ) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let callbacks: &Self = slf.get(); + if kind == ResultKind::SqlTest { + let test: TestResultText = test_text(result)?; + if test.outcome != PASS_OUTCOME { + Self::retire_live_row(slf)?; + return callbacks.emit( + py, + &render_failed_test_result(&test, callbacks.layout(py)?, callbacks.style), + ); + } + let mut state = callbacks.state(); + for step in &test.steps { + state + .tests_by_model + .entry(step.model_name.clone()) + .or_default() + .push(test.clone()); + } + return Ok(()); + } + Self::retire_live_row(slf)?; + let counter: usize = { + let mut state = callbacks.state(); + state.counter += 1; + state.counter + }; + let layout: ProgressLayout = callbacks.layout(py)?; + let style: CliStyle = callbacks.style; + let text: String = match kind { + ResultKind::Seed => render_resource_result( + ResourceResult::Node(&seed_text(result)?), + counter, + layout, + style, + ), + ResultKind::Function => render_resource_result( + ResourceResult::Node(&function_text(result)?), + counter, + layout, + style, + ), + ResultKind::Load => render_resource_result( + ResourceResult::Load(&load_text(result)?), + counter, + layout, + style, + ), + ResultKind::Model => { + let model = model_text(result)?; + let facts: &ModelRowFacts = callbacks + .facts + .models + .get(&model.name) + .unwrap_or(&callbacks.facts.unplanned_model); + let tests: Vec = callbacks + .state() + .tests_by_model + .get(&model.name) + .cloned() + .unwrap_or_default(); + render_model_result((&model, facts, &tests), counter, layout, style) + } + _ => return Ok(()), + }; + callbacks.emit(py, &text) + } + + pub(crate) fn write_event(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { + let writer: &Bound<'_, PyAny> = self.event_writer.bind(py); + let write_one = |item: &Bound<'_, PyAny>| -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("result", item)?; + options.set_item("plan", self.plan.bind(py))?; + writer.call_method("write_build_result", (), Some(&options))?; + Ok(()) + }; + write_one(result)?; + if result_kind(result)? == ResultKind::Model { + for audit in native_field(result, "audit_results")?.try_iter()? { + write_one(&audit?)?; + } + } + Ok(()) + } +} + +#[pymethods] +impl BuildProgressCallbacks { + #[new] + #[pyo3(signature = (**options))] + pub fn new(py: Python<'_>, options: Option>) -> PyResult { + let arguments = KeywordArguments::read( + py, + "BuildProgressCallbacks", + options, + ( + &["plan", "use_color"], + &["verbose", "debug", "event_output_path"], + ), + )?; + let flag = |name: &str| -> PyResult { + Ok(arguments.passed(name)? && arguments.value(name)?.is_truthy()?) + }; + let plan: Bound<'_, PyAny> = arguments.value("plan")?; + let debug: bool = flag("debug")?; + let sys: Bound<'_, PyAny> = py.import("sys")?.into_any(); + let stdout: Bound<'_, PyAny> = native_field(&sys, "stdout")?; + let stream: Bound<'_, PyAny> = if debug { + native_field(&sys, "stderr")? + } else { + stdout.clone() + }; + let path: Bound<'_, PyAny> = if arguments.passed("event_output_path")? { + arguments.value("event_output_path")? + } else { + py.None().into_bound(py) + }; + let event_writer: Bound<'_, PyAny> = Bound::new( + py, + ExecutionEventWriter::new(py, Some(path).filter(|value| !value.is_none()))?, + )? + .into_any(); + let facts: BuildPlanFacts = build_plan_facts(&plan)?; + let projector: Option> = projector(py)?; + if let Some(projector) = &projector { + let projector: &Bound<'_, PyAny> = projector.bind(py); + let ordinals: Bound<'_, PyDict> = PyDict::new(py); + for (name, ordinal) in &facts.ordinals { + ordinals.set_item(name, ordinal)?; + } + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("ordinals", ordinals)?; + options.set_item("total", facts.total)?; + projector.call_method("configure_resources", (), Some(&options))?; + for name in &facts.test_names { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("resource_name", name)?; + projector.call_method("expect_resource_enrichment", (), Some(&options))?; + } + } + let lines: Bound<'_, PyAny> = shared_lines(py)?; + Ok(Self { + worker: None, + plan: plan.unbind(), + is_tty: tty_flag(&stdout)? && !debug, + stream: stream.unbind(), + lock: native_field(&lines, "lock")?.unbind(), + lines: lines.unbind(), + event_writer: event_writer.unbind(), + projector, + facts, + style: CliStyle { + use_color: arguments.value("use_color")?.is_truthy()?, + }, + verbose: flag("verbose")?, + debug, + started: Instant::now(), + state: Mutex::new(CallbackState::default()), + }) + } + + #[getter] + pub fn elapsed(&self) -> f64 { + self.started.elapsed().as_secs_f64() + } + + #[pyo3(signature = (*, name, resource_kind))] + pub(crate) fn on_node_start( + slf: &Bound<'_, Self>, + name: String, + resource_kind: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let callbacks: &Self = slf.get(); + if let Some(worker) = &callbacks.worker { + worker.start(py, &name, resource_kind)?; + } + if let Some(projector) = &callbacks.projector { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("resource_name", name)?; + projector + .bind(py) + .call_method("expect_resource_enrichment", (), Some(&options))?; + return Ok(()); + } + { + let mut state = callbacks.state(); + state.node_name = name; + state.node_kind = native_field(resource_kind, "value")?.extract()?; + state.sub_message = String::new(); + } + if !callbacks.is_tty { + return Ok(()); + } + callbacks.set_cursor(py, true)?; + callbacks.locked(py, || { + callbacks.state().line_active = true; + claim( + callbacks.lines.bind(py), + callbacks.stream.bind(py), + slf.as_any(), + )?; + callbacks.write_spinner_line(py) + })?; + callbacks.stop_spinner(py)?; + let (event, thread) = start_spinner(slf.as_any())?; + let mut state = callbacks.state(); + state.stop_event = Some(event.unbind()); + state.thread = Some(thread.unbind()); + Ok(()) + } + + pub(crate) fn on_sub_progress(&self, py: Python<'_>, message: String) -> PyResult<()> { + self.state().sub_message = message; + if self.is_tty { + self.write_spinner_line(py)?; + } + Ok(()) + } + + /// Keep a line per completed microbatch when no live spinner row shows batch progress. + pub(crate) fn on_batch_complete( + &self, + py: Python<'_>, + progress: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.write_batch_complete(py, progress) + } + + /// Report one deduplicated concurrent scheduler state transition. + pub(crate) fn on_scheduler_state( + &self, + py: Python<'_>, + state: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let state: SchedulerStateText = scheduler_state(state)?; + { + let mut callback_state = self.state(); + callback_state.scheduler_count += 1; + if callback_state.scheduler_count > SCHEDULER_DIAGNOSTIC_LIMIT { + callback_state.scheduler_omitted += 1; + callback_state.last_scheduler = Some(state); + return Ok(()); + } + } + self.write_runtime_diagnostic(py, &scheduler_diagnostic_message(state)) + } + + /// Report a bounded sample of completed warehouse statement identities. + pub(crate) fn on_statement_complete( + &self, + py: Python<'_>, + telemetry: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let telemetry: StatementText = statement(telemetry)?; + { + let mut state = self.state(); + let count: &mut usize = state + .query_counts + .entry(telemetry.resource_name.clone()) + .or_default(); + *count += 1; + if *count > QUERY_DIAGNOSTIC_LIMIT { + state.query_omitted += 1; + state.last_query = Some(telemetry); + return Ok(()); + } + } + self.write_runtime_diagnostic(py, &query_diagnostic_message(&telemetry)) + } + + /// Erase the live spinner row so a persistent line can take its place. + pub(crate) fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || self.emit(py, CLEAR_LINE)) + } + + /// Draw the live spinner row again below persistent output. + pub(crate) fn redraw_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.write_spinner_line(py) + } + + pub(crate) fn on_node_complete( + slf: &Bound<'_, Self>, + node_result: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let callbacks: &Self = slf.get(); + if let Some(worker) = &callbacks.worker + && let Some((_, key)) = resource_identity(node_result)? + { + worker.completed(node_result, &key)?; + } + let kind: ResultKind = result_kind(node_result)?; + let mut result: Bound<'_, PyAny> = node_result.clone(); + if let Some(projector) = &callbacks.projector + && let Some((name, identity)) = resource_identity(node_result)? + { + let Some(duration_ms) = consume_terminal(projector.bind(py), &name, Some(&identity))? + else { + return Ok(()); + }; + if matches!( + kind, + ResultKind::Model | ResultKind::Seed | ResultKind::Function | ResultKind::Load + ) { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("duration_ms", duration_ms.trunc() as i64)?; + result = replaced_keywords(node_result, &options)?; + } + } + let written: PyResult<()> = Self::write_node_result(slf, &result, kind); + let evented: PyResult<()> = callbacks.write_event(py, &result); + written.and(evented) + } + + /// Write one result to the shared structured event channel. + pub(crate) fn write_execution_event( + &self, + py: Python<'_>, + result: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.write_event(py, result) + } + + pub(crate) fn on_check_start(&self, py: Python<'_>, name: &str) -> PyResult<()> { + if let Some(worker) = &self.worker { + worker.start_check(py, name)?; + } + Ok(()) + } + + pub(crate) fn on_check_complete( + &self, + py: Python<'_>, + result: &Bound<'_, PyAny>, + ) -> PyResult<()> { + if let Some(worker) = &self.worker { + worker.completed_check(result)?; + } + self.write_event(py, result) + } + + /// Close the optional structured execution event stream. + pub(crate) fn close(&self, py: Python<'_>) -> PyResult<()> { + let (scheduler, query) = { + let state = self.state(); + ( + state + .last_scheduler + .map(|last| (state.scheduler_omitted, last)), + state + .last_query + .clone() + .map(|last| (state.query_omitted, last)), + ) + }; + for line in runtime_diagnostic_summary( + scheduler, + query.as_ref().map(|(omitted, last)| (*omitted, last)), + ) { + self.write_runtime_diagnostic(py, &line)?; + } + self.event_writer.bind(py).call_method0("close")?; + Ok(()) + } + + pub(crate) fn _spin_until_stopped( + &self, + py: Python<'_>, + stop_event: &Bound<'_, PyAny>, + ) -> PyResult<()> { + while !stop_event + .call_method1("wait", (SPINNER_TICK_SECONDS,))? + .is_truthy()? + { + self.write_spinner_line(py)?; + } + Ok(()) + } +} + +/// Render one live spinner row for all running items and grouped completed rows. +#[pyclass(module = "sqlbuild._native", frozen)] +pub struct NestedCommandProgressCallbacks { + pub(crate) stream: Py, + pub(crate) lines: Py, + pub(crate) lock: Py, + pub(crate) label: String, + pub(crate) name_width: usize, + pub(crate) style: CliStyle, + pub(crate) is_tty: bool, + pub(crate) sequential: bool, + pub(crate) projector: Option>, + pub(crate) state: Mutex, +} + +impl std::fmt::Debug for NestedCommandProgressCallbacks { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("NestedCommandProgressCallbacks") + .finish_non_exhaustive() + } +} + +impl NestedCommandProgressCallbacks { + pub(crate) fn state( + &self, + ) -> MutexGuard<'_, crate::build_progress::_helpers::nested_progress::ReporterState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + pub(crate) fn locked( + &self, + py: Python<'_>, + body: impl FnOnce() -> PyResult, + ) -> PyResult { + within(self.lock.bind(py), |_| body()) + } + + pub(crate) fn layout(&self, py: Python<'_>) -> PyResult { + Ok(NestedLayout { + label: self.label.clone(), + name_width: self.name_width, + columns: terminal_columns(py)?, + }) + } + + pub(crate) fn draw_spinner_line(&self, py: Python<'_>) -> PyResult<()> { + let (running, frame) = { + let mut state = self.state(); + let frame: usize = state.frame; + state.frame = (frame + 1) % SPINNER_FRAMES.len(); + let running: Vec = state.running.iter().map(|(item, _)| item.clone()).collect(); + (running, frame) + }; + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write( + stream, + &render_nested_spinner_line( + &running, + SPINNER_FRAMES[frame], + &self.layout(py)?, + self.style, + ), + )?; + flush(stream) + } + + pub(crate) fn clear_live_row(&self, py: Python<'_>) -> PyResult<()> { + if self.is_tty && self.state().line_active { + write(self.stream.bind(py), CLEAR_LINE)?; + } + Ok(()) + } + + pub(crate) fn write_group_header(&self, py: Python<'_>, group: &str) -> PyResult<()> { + let previous: Option = self.state().printed_group.clone(); + if previous.as_deref() == Some(group) { + return Ok(()); + } + self.state().printed_group = Some(group.to_owned()); + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write( + stream, + &render_group_header(group, previous.is_none(), self.style), + )?; + flush(stream) + } + + pub(crate) fn set_cursor(&self, py: Python<'_>, hidden: bool) -> PyResult<()> { + if self.state().cursor_hidden == hidden { + return Ok(()); + } + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, if hidden { HIDE_CURSOR } else { SHOW_CURSOR })?; + flush(stream)?; + self.state().cursor_hidden = hidden; + Ok(()) + } + + /// Redraw the live row for remaining items, or retire it; return a thread to join. + pub(crate) fn refresh_live_row(slf: &Bound<'_, Self>) -> PyResult>> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + if !reporter.is_tty { + return Ok(None); + } + if !reporter.state().running.is_empty() { + reporter.draw_spinner_line(py)?; + return Ok(None); + } + reporter.state().line_active = false; + release(reporter.lines.bind(py), slf.as_any())?; + reporter.set_cursor(py, false)?; + let (event, thread) = { + let mut state = reporter.state(); + (state.stop_event.take(), state.thread.take()) + }; + if let Some(event) = event { + event.bind(py).call_method0("set")?; + } + Ok(thread) + } +} + +#[pymethods] +impl NestedCommandProgressCallbacks { + #[new] + #[pyo3(signature = (**options))] + pub fn new(py: Python<'_>, options: Option>) -> PyResult { + let arguments = KeywordArguments::read( + py, + "NestedCommandProgressCallbacks", + options, + ( + &["total", "label", "stream", "use_color"], + &["name_width", "concurrency"], + ), + )?; + let stream: Bound<'_, PyAny> = arguments.value("stream")?; + let lines: Bound<'_, PyAny> = shared_lines(py)?; + let name_width: usize = if arguments.passed("name_width")? { + arguments.value("name_width")?.extract()? + } else { + MIN_NAME_WIDTH + }; + let concurrency: i64 = if arguments.passed("concurrency")? { + arguments.value("concurrency")?.extract()? + } else { + 1 + }; + Ok(Self { + is_tty: tty_flag(&stream)?, + stream: stream.unbind(), + lock: native_field(&lines, "lock")?.unbind(), + lines: lines.unbind(), + label: arguments.value("label")?.extract()?, + name_width: name_width.max(MIN_NAME_WIDTH), + style: CliStyle { + use_color: arguments.value("use_color")?.is_truthy()?, + }, + sequential: concurrency <= 1, + projector: projector(py)?, + state: Mutex::new( + crate::build_progress::_helpers::nested_progress::ReporterState::default(), + ), + }) + } + + #[pyo3(signature = (**options))] + pub(crate) fn on_item_start( + slf: &Bound<'_, Self>, + options: Option>, + ) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + let arguments = KeywordArguments::read( + py, + "on_item_start", + options, + (&["group_name", "item_name"], &["canonical_resource_name"]), + )?; + let group: String = arguments.value("group_name")?.extract()?; + let item: String = arguments.value("item_name")?.extract()?; + if let Some(projector) = &reporter.projector { + let canonical: Option = + optional_argument(&arguments, "canonical_resource_name")?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item( + "resource_name", + canonical + .filter(|name| !name.is_empty()) + .unwrap_or(item.clone()), + )?; + projector + .bind(py) + .call_method("expect_resource_enrichment", (), Some(&options))?; + } + reporter.locked(py, || { + if reporter.sequential && reporter.state().running.is_empty() { + reporter.clear_live_row(py)?; + reporter.write_group_header(py, &group)?; + } + { + let mut state = reporter.state(); + match state.running.iter_mut().find(|(name, _)| *name == item) { + Some(entry) => entry.1 = group, + None => state.running.push((item, group)), + } + } + if !reporter.is_tty { + return Ok(()); + } + reporter.set_cursor(py, true)?; + reporter.state().line_active = true; + claim( + reporter.lines.bind(py), + reporter.stream.bind(py), + slf.as_any(), + )?; + reporter.draw_spinner_line(py)?; + if reporter.state().thread.is_none() { + let (event, thread) = start_spinner(slf.as_any())?; + let mut state = reporter.state(); + state.stop_event = Some(event.unbind()); + state.thread = Some(thread.unbind()); + } + Ok(()) + }) + } + + #[pyo3(signature = (**options))] + pub(crate) fn on_item_complete( + slf: &Bound<'_, Self>, + options: Option>, + ) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let reporter: &Self = slf.get(); + let arguments = KeywordArguments::read(py, "on_item_complete", options, ITEM_OPTIONS)?; + let group: String = arguments.value("group_name")?.extract()?; + let item_name: String = arguments.value("item_name")?.extract()?; + let mut detail: String = optional_argument(&arguments, "detail")?.unwrap_or_default(); + let duration_ms: Option = match &reporter.projector { + Some(projector) => { + let canonical: Option = + optional_argument(&arguments, "canonical_resource_name")?; + consume_terminal( + projector.bind(py), + &canonical + .filter(|name| !name.is_empty()) + .unwrap_or(item_name.clone()), + optional_argument(&arguments, "canonical_resource_id")?.as_deref(), + )? + } + None => None, + }; + let stopped: Option> = reporter.locked(py, || { + let item_group: String = { + let mut state = reporter.state(); + match state + .running + .iter() + .position(|(name, _)| *name == item_name) + { + Some(index) => state.running.remove(index).1, + None => group, + } + }; + if reporter.projector.is_none() || duration_ms.is_some() { + if let Some(duration_ms) = duration_ms { + detail = format!("{detail} {:.2}s", duration_ms / 1000.0); + } + let item: NestedItem = NestedItem { + item_name, + status_text: optional_argument(&arguments, "status_text")?.unwrap_or_default(), + detail, + error: ErrorText { + code: optional_argument(&arguments, "error_code")?, + message: optional_argument(&arguments, "error_message")?, + help: optional_argument(&arguments, "error_help")?, + }, + child_rows: child_rows(&arguments)?, + }; + reporter.clear_live_row(py)?; + reporter.write_group_header(py, &item_group)?; + write( + reporter.stream.bind(py), + &render_nested_item(&item, &reporter.layout(py)?, reporter.style), + )?; + } + let stopped: Option> = Self::refresh_live_row(slf)?; + flush(reporter.stream.bind(py))?; + Ok(stopped) + })?; + if let Some(thread) = stopped { + join_spinner(thread.bind(py))?; + } + Ok(()) + } + + /// Erase the live spinner row so a persistent line can take its place. + pub(crate) fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, CLEAR_LINE)?; + flush(stream) + }) + } + + /// Draw the live spinner row again below persistent output. + pub(crate) fn redraw_transient_line(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + if self.state().line_active { + self.draw_spinner_line(py)?; + } + Ok(()) + }) + } + + pub(crate) fn _spin_until_stopped( + &self, + py: Python<'_>, + stop_event: &Bound<'_, PyAny>, + ) -> PyResult<()> { + while !stop_event + .call_method1("wait", (SPINNER_TICK_SECONDS,))? + .is_truthy()? + { + self.locked(py, || { + if !stop_event.call_method0("is_set")?.is_truthy()? && self.state().line_active { + self.draw_spinner_line(py)?; + } + Ok(()) + })?; + } + Ok(()) + } +} + +/// Render simple connection start, success, and failure messages. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct ConnectionProgressReporter { + pub(crate) adapter_name: String, + pub(crate) stream: Py, + pub(crate) blank_line_before_start: bool, + pub(crate) blank_line_after_complete: bool, + pub(crate) style: CliStyle, +} + +impl ConnectionProgressReporter { + pub(crate) fn write(&self, py: Python<'_>, message: &str) -> PyResult<()> { + write_persistent(py, self.stream.bind(py), &format!("{message}\n")) + } +} + +#[pymethods] +impl ConnectionProgressReporter { + #[new] + #[pyo3(signature = (**options))] + pub fn new(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "ConnectionProgressReporter", + options, + CONNECTION_OPTIONS, + )?; + Ok(Self { + adapter_name: arguments.value("adapter_name")?.extract()?, + stream: arguments.value("stream")?.unbind(), + blank_line_before_start: arguments.flag("blank_line_before_start")?, + blank_line_after_complete: arguments.flag("blank_line_after_complete")?, + style: CliStyle { + use_color: arguments.flag("use_color")?, + }, + }) + } + + pub(crate) fn on_connection_start( + &self, + py: Python<'_>, + connection_count: i64, + ) -> PyResult<()> { + if self.blank_line_before_start { + self.write(py, "")?; + } + let message: String = connection_start_message(&self.adapter_name, connection_count); + self.write(py, &self.style.muted(&message)) + } + + #[pyo3(signature = (*, connection_count, elapsed_seconds))] + pub(crate) fn on_connection_complete( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + let _ = connection_count; + let line: String = + connection_phase_line(self.style, true, (&self.adapter_name, elapsed_seconds)); + self.write(py, &line)?; + if self.blank_line_after_complete { + self.write(py, "")?; + } + Ok(()) + } + + #[pyo3(signature = (*, connection_count, elapsed_seconds))] + pub(crate) fn on_connection_error( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + let _ = connection_count; + let line: String = + connection_phase_line(self.style, false, (&self.adapter_name, elapsed_seconds)); + self.write(py, &line) + } +} + +/// Render simple planning phase progress messages. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct PlanningProgressReporter { + pub(crate) status: Py, + pub(crate) active: AtomicBool, + pub(crate) projector: Option>, +} + +impl PlanningProgressReporter { + pub(crate) fn call_status(&self, py: Python<'_>, method: &str, message: &str) -> PyResult<()> { + self.status.bind(py).call_method1(method, (message,))?; + Ok(()) + } + + pub(crate) fn call_status_keyword( + &self, + py: Python<'_>, + method: &str, + message: &str, + ) -> PyResult<()> { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("message", message)?; + self.status + .bind(py) + .call_method(method, (), Some(&keywords))?; + Ok(()) + } +} + +#[pymethods] +impl PlanningProgressReporter { + #[new] + #[pyo3(signature = (*, stream, use_color = false))] + pub fn new(py: Python<'_>, stream: &Bound<'_, PyAny>, use_color: bool) -> PyResult { + let status: Bound<'_, PyAny> = status_reporter(stream, use_color)?; + Ok(Self { + status: status.unbind(), + active: AtomicBool::new(false), + projector: current_projector(py)?, + }) + } + + pub fn on_progress(&self, py: Python<'_>, message: &str) -> PyResult<()> { + if let Some(projector) = &self.projector + && projector_operation_active(projector.get(), PROJECT_COMPILE_OPERATION)? + { + return Ok(()); + } + if is_planning_completion_message(message) { + return self.complete(py, message); + } + if is_planning_failure_message(message) { + return self.error(py, message); + } + if !self.active.load(Ordering::SeqCst) { + return self.start(py, message); + } + self.update(py, message) + } + + pub fn start(&self, py: Python<'_>, message: &str) -> PyResult<()> { + self.call_status(py, "start", message)?; + self.active.store(true, Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn update(&self, py: Python<'_>, message: &str) -> PyResult<()> { + if !self.active.load(Ordering::SeqCst) { + return self.start(py, message); + } + self.call_status(py, "update", message) + } + + pub fn complete(&self, py: Python<'_>, message: &str) -> PyResult<()> { + self.call_status_keyword(py, "complete", message)?; + self.active.store(false, Ordering::SeqCst); + Ok(()) + } + + /// Complete with a pre-styled message written verbatim. + pub fn complete_styled(&self, py: Python<'_>, message: &str) -> PyResult<()> { + self.call_status_keyword(py, "complete_styled", message)?; + self.active.store(false, Ordering::SeqCst); + Ok(()) + } + + /// Close active progress and render a non-success terminal message. + pub(crate) fn error(&self, py: Python<'_>, message: &str) -> PyResult<()> { + self.call_status(py, "error", message)?; + self.active.store(false, Ordering::SeqCst); + Ok(()) + } + + #[pyo3(signature = (*, blank_line_after = false))] + pub fn finish(&self, py: Python<'_>, blank_line_after: bool) -> PyResult<()> { + self.status.bind(py).call_method0("close")?; + self.active.store(false, Ordering::SeqCst); + if blank_line_after { + self.status.bind(py).call_method0("write_blank_line")?; + } + Ok(()) + } +} + +/// Render timed `sqb dbt init` phase progress. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct DbtInitProgressReporter { + pub(crate) stream: Py, + pub(crate) status: Py, + pub(crate) started: AtomicBool, + pub(crate) phase_start: Mutex>, +} + +#[pymethods] +impl DbtInitProgressReporter { + #[new] + #[pyo3(signature = (*, stream, use_color))] + pub fn new(stream: &Bound<'_, PyAny>, use_color: bool) -> PyResult { + Ok(Self { + stream: stream.clone().unbind(), + status: status_reporter(stream, use_color)?.unbind(), + started: AtomicBool::new(false), + phase_start: Mutex::new(None), + }) + } + + pub(crate) fn start(&self, py: Python<'_>, message: &str) -> PyResult<()> { + if !self.started.load(Ordering::SeqCst) { + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + stream.call_method1("write", ("\n",))?; + stream.call_method0("flush")?; + self.started.store(true, Ordering::SeqCst); + } + *self + .phase_start + .lock() + .unwrap_or_else(PoisonError::into_inner) = Some(Instant::now()); + self.status.bind(py).call_method1("start", (message,))?; + Ok(()) + } + + pub(crate) fn complete(&self, py: Python<'_>, message: &str) -> PyResult<()> { + let phase_start: Option = self + .phase_start + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + let elapsed_seconds: f64 = phase_start.map_or(0.0, |start| start.elapsed().as_secs_f64()); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("message", format!("{message} ({elapsed_seconds:.2}s)"))?; + self.status + .bind(py) + .call_method("complete", (), Some(&keywords))?; + Ok(()) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs b/crates/sqlbuild-python-output/src/compute_logs/_helpers/conversion.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs rename to crates/sqlbuild-python-output/src/compute_logs/_helpers/conversion.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/getters.rs b/crates/sqlbuild-python-output/src/compute_logs/_helpers/getters.rs similarity index 55% rename from crates/sqlbuild-python/src/bindings/_helpers/compute_logs/getters.rs rename to crates/sqlbuild-python-output/src/compute_logs/_helpers/getters.rs index 11c5b10ca5..3c0e9c7f90 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/getters.rs +++ b/crates/sqlbuild-python-output/src/compute_logs/_helpers/getters.rs @@ -3,17 +3,9 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::compute_logs::storage::LocalFilesystemComputeLogStorage; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::compute_logs::_helpers::storage::LocalFilesystemComputeLogStorage; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { - class: class_of::, - read: read_local_filesystem_compute_log_storage, -}]; - -fn read_local_filesystem_compute_log_storage<'py>( +pub(crate) fn read_local_filesystem_compute_log_storage<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-output/src/compute_logs/_helpers/mod.rs b/crates/sqlbuild-python-output/src/compute_logs/_helpers/mod.rs new file mode 100644 index 0000000000..2bb4d7344a --- /dev/null +++ b/crates/sqlbuild-python-output/src/compute_logs/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod conversion; +pub(crate) mod getters; +pub(crate) mod storage; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs b/crates/sqlbuild-python-output/src/compute_logs/_helpers/storage.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs rename to crates/sqlbuild-python-output/src/compute_logs/_helpers/storage.rs index bdd3ca65c3..939ff34709 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs +++ b/crates/sqlbuild-python-output/src/compute_logs/_helpers/storage.rs @@ -30,12 +30,17 @@ use sqlbuild_cli::compute_logs::models::CaptureDocument; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_recall::writer::models::RecallSession; -use crate::bindings::_helpers::compute_logs::conversion::{ - capture_id, document, inventory_item, is_exact_int, metadata_model, rejection, storage_error, - stream_index, -}; -use crate::bindings::_helpers::recall::session::{forward_diagnostic, forward_output}; -use crate::bindings::_helpers::recall::sinks::RecallLifecycleSink; +use crate::compute_logs::_helpers::conversion::capture_id; +use crate::compute_logs::_helpers::conversion::document; +use crate::compute_logs::_helpers::conversion::inventory_item; +use crate::compute_logs::_helpers::conversion::is_exact_int; +use crate::compute_logs::_helpers::conversion::metadata_model; +use crate::compute_logs::_helpers::conversion::rejection; +use crate::compute_logs::_helpers::conversion::storage_error; +use crate::compute_logs::_helpers::conversion::stream_index; +use crate::recall::main::forward_diagnostic::forward_diagnostic; +use crate::recall::main::forward_output::forward_output; +use crate::recall::models::RecallLifecycleSink; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; diff --git a/crates/sqlbuild-python-output/src/compute_logs/constants.rs b/crates/sqlbuild-python-output/src/compute_logs/constants.rs new file mode 100644 index 0000000000..d6d59fe90e --- /dev/null +++ b/crates/sqlbuild-python-output/src/compute_logs/constants.rs @@ -0,0 +1,12 @@ +//! Public constants of compute logs. + +use crate::compute_logs::_helpers::getters::read_local_filesystem_compute_log_storage; +use crate::compute_logs::_helpers::storage::LocalFilesystemComputeLogStorage; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[GetterEntry { + class: class_of::, + read: read_local_filesystem_compute_log_storage, +}]; diff --git a/crates/sqlbuild-python-output/src/compute_logs/main/mod.rs b/crates/sqlbuild-python-output/src/compute_logs/main/mod.rs new file mode 100644 index 0000000000..38814cd6d8 --- /dev/null +++ b/crates/sqlbuild-python-output/src/compute_logs/main/mod.rs @@ -0,0 +1 @@ +pub mod register_storage; diff --git a/crates/sqlbuild-python-output/src/compute_logs/main/register_storage.rs b/crates/sqlbuild-python-output/src/compute_logs/main/register_storage.rs new file mode 100644 index 0000000000..ea840817ec --- /dev/null +++ b/crates/sqlbuild-python-output/src/compute_logs/main/register_storage.rs @@ -0,0 +1,9 @@ +//! Register the local filesystem compute log storage class. + +use crate::compute_logs::_helpers::storage::LocalFilesystemComputeLogStorage; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the local filesystem compute log storage class. +pub fn register_storage(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python-output/src/compute_logs/mod.rs b/crates/sqlbuild-python-output/src/compute_logs/mod.rs new file mode 100644 index 0000000000..316e7cab96 --- /dev/null +++ b/crates/sqlbuild-python-output/src/compute_logs/mod.rs @@ -0,0 +1,5 @@ +//! Compute log capture storage on the local filesystem and its Python conversions. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs b/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/allocation.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs rename to crates/sqlbuild-python-output/src/cost_telemetry/_helpers/allocation.rs index ff7d020b11..fd85a02187 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs +++ b/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/allocation.rs @@ -1,9 +1,9 @@ //! `allocate_run_cost`: price one run's share of observed Snowflake warehouse busy time. use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; use pyo3::types::{PyDict, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cost_allocation::main::allocate_run_cost::allocate_run_cost as allocate; use sqlbuild_core::cost_allocation::models::{ AllocationStatus, CostQuery, PyDecimal, ResourceAllocation, RunAllocation, WarehouseKey, @@ -196,7 +196,7 @@ pub(crate) fn allocate_observations( /// Concurrency-safe Snowflake busy-compute attribution for one run. #[pyfunction] #[pyo3(signature = (**options))] -fn allocate_run_cost<'py>( +pub(crate) fn allocate_run_cost<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -227,8 +227,3 @@ fn allocate_run_cost<'py>( (&usd_per_credit, &arguments.value("rate_source")?), ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(allocate_run_cost, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs b/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/health.rs similarity index 84% rename from crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs rename to crates/sqlbuild-python-output/src/cost_telemetry/_helpers/health.rs index 9539893b06..5f13c241d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs +++ b/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/health.rs @@ -3,14 +3,14 @@ use std::collections::HashMap; use std::sync::{Mutex, MutexGuard, PoisonError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::PyTypeMethods; use pyo3::{pyclass, pymethods}; use sqlbuild_python_front::cost_context::main::process_object::process_object; -const HEALTH_ATTRIBUTE: &str = "COST_TELEMETRY_HEALTH"; +pub(crate) const HEALTH_ATTRIBUTE: &str = "COST_TELEMETRY_HEALTH"; static HEALTH: PyOnceLock> = PyOnceLock::new(); @@ -45,7 +45,7 @@ impl CostTelemetryHealth { } } -fn health(py: Python<'_>) -> PyResult> { +pub(crate) fn health(py: Python<'_>) -> PyResult> { let tracker: &Bound<'_, PyAny> = process_object(&HEALTH, py, || { Ok(Bound::new( py, @@ -67,8 +67,3 @@ pub(crate) fn mark_ledger_failure(run_id: String, error: &Bound<'_, PyAny>) -> P pub(crate) fn consume_ledger_failure(py: Python<'_>, run_id: &str) -> PyResult> { Ok(health(py)?.get().consume_ledger_failure(run_id)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add(HEALTH_ATTRIBUTE, health(module.py())?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs b/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/ledger.rs similarity index 94% rename from crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs rename to crates/sqlbuild-python-output/src/cost_telemetry/_helpers/ledger.rs index b02639c427..32ab3cf809 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs +++ b/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/ledger.rs @@ -5,17 +5,15 @@ use std::io::Write; use std::path::{Path, PathBuf}; use pyo3::exceptions::{PyKeyError, PyTypeError, PyValueError}; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; use pyo3::sync::PyOnceLock; use pyo3::types::{PyBytes, PyDict, PyString, PyTuple, PyTypeMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; use sqlbuild_core::json::main::dumps::dumps; use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; -use crate::bindings::_helpers::cost_telemetry::health::mark_ledger_failure; +use crate::cost_telemetry::_helpers::health::mark_ledger_failure; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -129,7 +127,7 @@ fn append_line(path: &Path, line: &str) -> std::io::Result<()> { /// Append statement identity without allowing telemetry I/O to affect execution. #[pyfunction] #[pyo3(signature = (**options))] -fn record_statement(py: Python<'_>, options: Option>) -> PyResult<()> { +pub(crate) fn record_statement(py: Python<'_>, options: Option>) -> PyResult<()> { let arguments: KeywordArguments<'_> = KeywordArguments::read( py, "record_statement", @@ -253,15 +251,9 @@ pub(crate) fn read_ledger<'py>( /// Read a stable snapshot of valid statement ledger entries. #[pyfunction] #[pyo3(signature = (*, path, run_id))] -fn read_statement_ledger<'py>( +pub(crate) fn read_statement_ledger<'py>( path: &Bound<'py, PyAny>, run_id: &Bound<'py, PyAny>, ) -> PyResult> { PyTuple::new(path.py(), read_ledger(path, run_id)?) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(record_statement, module)?)?; - module.add_function(wrap_pyfunction!(read_statement_ledger, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/mod.rs b/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/mod.rs new file mode 100644 index 0000000000..cd06a31fe3 --- /dev/null +++ b/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod allocation; +pub(crate) mod health; +pub(crate) mod ledger; +pub(crate) mod snowflake; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs b/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs rename to crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs index 708bc7c78b..78824e1588 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs +++ b/crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs @@ -3,11 +3,9 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use pyo3::exceptions::{PyTimeoutError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyCFunction, PyDict, PyList, PyListMethods, PyString, PyStringMethods, PyTuple}; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_analysis::snowflake_cost::main::metadata_only_statement::metadata_only_statement; use sqlbuild_analysis::snowflake_cost::main::sqlbuild_query_tag::sqlbuild_query_tag; use sqlbuild_analysis::snowflake_cost::models::QueryTag; @@ -16,11 +14,12 @@ use sqlbuild_core::cost_allocation::main::query_history_sql::query_history_sql; use sqlbuild_core::cost_allocation::models::{QueryHistoryWindow, RunAllocation}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::cost_telemetry::allocation::{ - SummaryState, allocate_observations, priced, summary_object, -}; -use crate::bindings::_helpers::cost_telemetry::health::consume_ledger_failure; -use crate::bindings::_helpers::cost_telemetry::ledger::read_ledger; +use crate::cost_telemetry::_helpers::allocation::SummaryState; +use crate::cost_telemetry::_helpers::allocation::allocate_observations; +use crate::cost_telemetry::_helpers::allocation::priced; +use crate::cost_telemetry::_helpers::allocation::summary_object; +use crate::cost_telemetry::_helpers::health::consume_ledger_failure; +use crate::cost_telemetry::_helpers::ledger::read_ledger; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -744,7 +743,7 @@ fn collect<'py>( /// Collect visible history and attribute exact ledger query IDs to one run. #[pyfunction] #[pyo3(signature = (**options))] -fn collect_snowflake_cost<'py>( +pub(crate) fn collect_snowflake_cost<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -818,7 +817,7 @@ fn collect_snowflake_cost<'py>( /// Query-history observations of one window, as cost collection reads them. #[pyfunction] #[pyo3(signature = (**options))] -fn snowflake_query_observations<'py>( +pub(crate) fn snowflake_query_observations<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -888,7 +887,7 @@ fn snowflake_query_observations<'py>( /// The query-history SQL of one window; an absent end reads every query ending after the start. #[pyfunction] #[pyo3(signature = (**options))] -fn render_snowflake_query_history_sql( +pub(crate) fn render_snowflake_query_history_sql( py: Python<'_>, options: Option>, ) -> PyResult { @@ -930,7 +929,7 @@ fn render_snowflake_query_history_sql( /// Whether Snowflake runs a statement of this type and text without warehouse compute. #[pyfunction] #[pyo3(signature = (*, query_type, query_text))] -fn snowflake_statement_is_metadata_only( +pub(crate) fn snowflake_statement_is_metadata_only( query_type: &Bound<'_, PyAny>, query_text: &Bound<'_, PyAny>, ) -> PyResult { @@ -949,7 +948,7 @@ fn snowflake_statement_is_metadata_only( /// The run attribution of a SQLBuild query tag, or `None` for any other tag. #[pyfunction] -fn parse_sqlbuild_query_tag<'py>( +pub(crate) fn parse_sqlbuild_query_tag<'py>( value: &Bound<'py, PyAny>, ) -> PyResult>> { let py: Python<'py> = value.py(); @@ -965,18 +964,3 @@ fn parse_sqlbuild_query_tag<'py>( attribution.set_item("resource_name", tag.resource_name)?; Ok(Some(attribution)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(collect_snowflake_cost, module)?)?; - module.add_function(wrap_pyfunction!(snowflake_query_observations, module)?)?; - module.add_function(wrap_pyfunction!( - render_snowflake_query_history_sql, - module - )?)?; - module.add_function(wrap_pyfunction!( - snowflake_statement_is_metadata_only, - module - )?)?; - module.add_function(wrap_pyfunction!(parse_sqlbuild_query_tag, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-output/src/cost_telemetry/main/mod.rs b/crates/sqlbuild-python-output/src/cost_telemetry/main/mod.rs new file mode 100644 index 0000000000..cf93274cd6 --- /dev/null +++ b/crates/sqlbuild-python-output/src/cost_telemetry/main/mod.rs @@ -0,0 +1,4 @@ +pub mod register_allocation; +pub mod register_health; +pub mod register_ledger; +pub mod register_snowflake; diff --git a/crates/sqlbuild-python-output/src/cost_telemetry/main/register_allocation.rs b/crates/sqlbuild-python-output/src/cost_telemetry/main/register_allocation.rs new file mode 100644 index 0000000000..3dcd757799 --- /dev/null +++ b/crates/sqlbuild-python-output/src/cost_telemetry/main/register_allocation.rs @@ -0,0 +1,11 @@ +//! Register the allocation functions and classes. + +use crate::cost_telemetry::_helpers::allocation::allocate_run_cost; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the allocation functions and classes. +pub fn register_allocation(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(allocate_run_cost, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-output/src/cost_telemetry/main/register_health.rs b/crates/sqlbuild-python-output/src/cost_telemetry/main/register_health.rs new file mode 100644 index 0000000000..4dbe934ac9 --- /dev/null +++ b/crates/sqlbuild-python-output/src/cost_telemetry/main/register_health.rs @@ -0,0 +1,12 @@ +//! Register the health functions and classes. + +use crate::cost_telemetry::_helpers::health::CostTelemetryHealth; +use crate::cost_telemetry::_helpers::health::HEALTH_ATTRIBUTE; +use crate::cost_telemetry::_helpers::health::health; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the health functions and classes. +pub fn register_health(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add(HEALTH_ATTRIBUTE, health(module.py())?) +} diff --git a/crates/sqlbuild-python-output/src/cost_telemetry/main/register_ledger.rs b/crates/sqlbuild-python-output/src/cost_telemetry/main/register_ledger.rs new file mode 100644 index 0000000000..a6d1d4dafb --- /dev/null +++ b/crates/sqlbuild-python-output/src/cost_telemetry/main/register_ledger.rs @@ -0,0 +1,13 @@ +//! Register the ledger functions and classes. + +use crate::cost_telemetry::_helpers::ledger::read_statement_ledger; +use crate::cost_telemetry::_helpers::ledger::record_statement; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the ledger functions and classes. +pub fn register_ledger(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(record_statement, module)?)?; + module.add_function(wrap_pyfunction!(read_statement_ledger, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-output/src/cost_telemetry/main/register_snowflake.rs b/crates/sqlbuild-python-output/src/cost_telemetry/main/register_snowflake.rs new file mode 100644 index 0000000000..b1021fb769 --- /dev/null +++ b/crates/sqlbuild-python-output/src/cost_telemetry/main/register_snowflake.rs @@ -0,0 +1,25 @@ +//! Register the snowflake functions and classes. + +use crate::cost_telemetry::_helpers::snowflake::collect_snowflake_cost; +use crate::cost_telemetry::_helpers::snowflake::parse_sqlbuild_query_tag; +use crate::cost_telemetry::_helpers::snowflake::render_snowflake_query_history_sql; +use crate::cost_telemetry::_helpers::snowflake::snowflake_query_observations; +use crate::cost_telemetry::_helpers::snowflake::snowflake_statement_is_metadata_only; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the snowflake functions and classes. +pub fn register_snowflake(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(collect_snowflake_cost, module)?)?; + module.add_function(wrap_pyfunction!(snowflake_query_observations, module)?)?; + module.add_function(wrap_pyfunction!( + render_snowflake_query_history_sql, + module + )?)?; + module.add_function(wrap_pyfunction!( + snowflake_statement_is_metadata_only, + module + )?)?; + module.add_function(wrap_pyfunction!(parse_sqlbuild_query_tag, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-output/src/cost_telemetry/mod.rs b/crates/sqlbuild-python-output/src/cost_telemetry/mod.rs new file mode 100644 index 0000000000..eaf7daf700 --- /dev/null +++ b/crates/sqlbuild-python-output/src/cost_telemetry/mod.rs @@ -0,0 +1,4 @@ +//! Cost telemetry: the cost ledger, its health, cost allocation and Snowflake cost reads. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python-output/src/lib.rs b/crates/sqlbuild-python-output/src/lib.rs new file mode 100644 index 0000000000..10d1540e1b --- /dev/null +++ b/crates/sqlbuild-python-output/src/lib.rs @@ -0,0 +1,10 @@ +//! The output layer of the `sqlbuild._native` bindings: build progress, result output, output capture, cost telemetry, compute logs and the Recall session. + +#![forbid(unsafe_code)] + +pub mod build_progress; +pub mod compute_logs; +pub mod cost_telemetry; +pub mod output_capture; +pub mod recall; +pub mod result_output; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs b/crates/sqlbuild-python-output/src/output_capture/_helpers/context.rs similarity index 82% rename from crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs rename to crates/sqlbuild-python-output/src/output_capture/_helpers/context.rs index 522d9a7535..5c27f98b91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs +++ b/crates/sqlbuild-python-output/src/output_capture/_helpers/context.rs @@ -4,8 +4,7 @@ use std::sync::Mutex; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyclass, - pyfunction, pymethods, wrap_pyfunction, + Bound, Py, PyAny, PyAnyMethods, PyResult, Python, pyclass, pyfunction, pymethods, }; use pyo3::sync::PyOnceLock; use pyo3::types::PyTuple; @@ -64,7 +63,7 @@ impl CommandOutputContext { /// Attach opaque integration context to command-output records. #[pyfunction] #[pyo3(signature = (*, external_context))] -fn command_output_context(external_context: Py) -> CommandOutputContext { +pub(crate) fn command_output_context(external_context: Py) -> CommandOutputContext { CommandOutputContext { external_context, token: Mutex::new(None), @@ -78,12 +77,6 @@ pub(crate) fn current_context(py: Python<'_>) -> PyResult> { /// Return optional integration context for the active command. #[pyfunction] -fn current_output_capture_context(py: Python<'_>) -> PyResult> { +pub(crate) fn current_output_capture_context(py: Python<'_>) -> PyResult> { current_context(py) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(command_output_context, module)?)?; - module.add_function(wrap_pyfunction!(current_output_capture_context, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs b/crates/sqlbuild-python-output/src/output_capture/_helpers/dispatcher.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs rename to crates/sqlbuild-python-output/src/output_capture/_helpers/dispatcher.rs index 3531325d57..afaa243aa2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs +++ b/crates/sqlbuild-python-output/src/output_capture/_helpers/dispatcher.rs @@ -6,8 +6,7 @@ use std::time::Instant; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, pyclass, - pyfunction, pymethods, wrap_pyfunction, + Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyclass, pyfunction, pymethods, }; use pyo3::types::{PyDict, PyString, PyTuple}; use sqlbuild_core::command_output::constants::{ @@ -23,9 +22,12 @@ use sqlbuild_core::command_output::models::{ }; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::output_capture::values::{ - freeze, object, output_error, pass_text, python_text, unencodable, -}; +use crate::output_capture::_helpers::values::freeze; +use crate::output_capture::_helpers::values::object; +use crate::output_capture::_helpers::values::output_error; +use crate::output_capture::_helpers::values::pass_text; +use crate::output_capture::_helpers::values::python_text; +use crate::output_capture::_helpers::values::unencodable; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -122,7 +124,7 @@ fn delivering_thread(py: Python<'_>) -> PyResult { /// Stop every capture worker and join it before finalization ends it inside native code. #[pyfunction] -fn stop_output_capture_threads(py: Python<'_>) -> PyResult<()> { +pub(crate) fn stop_output_capture_threads(py: Python<'_>) -> PyResult<()> { signal_interpreter_exit(); let mut threads: Vec> = Vec::new(); for thread in imported(py, "threading", "enumerate")? @@ -559,12 +561,3 @@ impl OutputCaptureDispatcher { ) } } - -/// Add the capture classes and run `stop_output_capture_threads` at interpreter exit. -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - let hook = wrap_pyfunction!(stop_output_capture_threads, module)?; - module.add_function(hook.clone())?; - imported(module.py(), "atexit", "register")?.call1((hook,))?; - module.add_class::()?; - module.add_class::() -} diff --git a/crates/sqlbuild-python-output/src/output_capture/_helpers/mod.rs b/crates/sqlbuild-python-output/src/output_capture/_helpers/mod.rs new file mode 100644 index 0000000000..d2c7f18637 --- /dev/null +++ b/crates/sqlbuild-python-output/src/output_capture/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod context; +pub(crate) mod dispatcher; +pub(crate) mod records; +pub(crate) mod text; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs b/crates/sqlbuild-python-output/src/output_capture/_helpers/records.rs similarity index 80% rename from crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs rename to crates/sqlbuild-python-output/src/output_capture/_helpers/records.rs index 600262a005..c8037f0ed0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs +++ b/crates/sqlbuild-python-output/src/output_capture/_helpers/records.rs @@ -1,14 +1,12 @@ //! Command-output record validation, wire JSON and the integration's invocation context. +use crate::output_capture::main::invocation_context::invocation_context; use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, pyfunction, - wrap_pyfunction, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyString, PyStringMethods}; use sqlbuild_core::command_output::constants::{ COMMAND_OUTPUT_LOSS_RECORD_TYPE, COMMAND_OUTPUT_RECORD_TYPE, CURRENT_SCHEMA_VERSION, - INVOCATION_CONTEXT_ENV, MAX_INVOCATION_CONTEXT_BYTES, MAX_INVOCATION_CONTEXT_DEPTH, + INVOCATION_CONTEXT_ENV, MAX_INVOCATION_CONTEXT_DEPTH, }; use sqlbuild_core::command_output::main::validate_output_record::validate_output_record; use sqlbuild_core::command_output::models::RecordShape; @@ -16,14 +14,16 @@ use sqlbuild_core::json::main::dumps::dumps; use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::output_capture::values::{ - freeze, json_value, object, output_error, plain_int, validation_error, -}; +use crate::output_capture::_helpers::values::freeze; +use crate::output_capture::_helpers::values::json_value; +use crate::output_capture::_helpers::values::object; +use crate::output_capture::_helpers::values::output_error; +use crate::output_capture::_helpers::values::plain_int; +use crate::output_capture::_helpers::values::validation_error; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; const ENVELOPE_FIELDS: [&str; 16] = [ "record_id", @@ -273,7 +273,7 @@ pub(crate) fn command_output_from_json<'py>( Ok(record) } -fn validate_depth(value: &Bound<'_, PyAny>, depth: usize) -> Result<(), PyErr> { +pub(crate) fn validate_depth(value: &Bound<'_, PyAny>, depth: usize) -> Result<(), PyErr> { if depth > MAX_INVOCATION_CONTEXT_DEPTH { return Err(output_error( value.py(), @@ -295,56 +295,10 @@ fn validate_depth(value: &Bound<'_, PyAny>, depth: usize) -> Result<(), PyErr> { Ok(()) } -fn input_error(py: Python<'_>, message: &str) -> PyErr { +pub(crate) fn input_error(py: Python<'_>, message: &str) -> PyErr { output_error(py, ExecutorHost::OutputCaptureInputError, message) } -/// Validated opaque invocation metadata supplied by an integration through the environment. -pub(crate) fn invocation_context<'py>( - py: Python<'py>, - environment: Option>, -) -> PyResult> { - let environment = match environment { - Some(environment) if !environment.is_none() => environment, - _ => imported(py, "os", "environ")?, - }; - let raw = mapping_get(&environment, INVOCATION_CONTEXT_ENV)?; - if raw.is_none() { - return Ok(PyDict::new(py).into_any()); - } - let text: String = raw.extract()?; - if text.len() > MAX_INVOCATION_CONTEXT_BYTES { - return Err(input_error( - py, - &format!( - "{INVOCATION_CONTEXT_ENV} must not exceed {MAX_INVOCATION_CONTEXT_BYTES} bytes" - ), - )); - } - let loaded = imported(py, "json", "loads")? - .call1((text,)) - .map_err(|error| match error.is_instance_of::(py) { - true => input_error( - py, - &format!("{INVOCATION_CONTEXT_ENV} must contain valid JSON"), - ), - false => error, - })?; - if !loaded.is_instance_of::() { - return Err(input_error( - py, - &format!("{INVOCATION_CONTEXT_ENV} must contain a JSON object"), - )); - } - validate_depth(&loaded, 1)?; - freeze(&loaded, "external_context").map_err(|error| { - match error.is_instance_of::(py) { - true => input_error(py, &error.value(py).to_string()), - false => error, - } - }) -} - /// Return validated opaque invocation metadata supplied by an integration. #[pyfunction] #[pyo3(signature = (*, environment = None))] @@ -354,13 +308,3 @@ pub(crate) fn invocation_context_from_environment<'py>( ) -> PyResult> { invocation_context(py, environment) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(freeze_command_output_record, module)?)?; - module.add_function(wrap_pyfunction!(command_output_to_json, module)?)?; - module.add_function(wrap_pyfunction!(command_output_from_json, module)?)?; - module.add_function(wrap_pyfunction!( - invocation_context_from_environment, - module - )?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/text.rs b/crates/sqlbuild-python-output/src/output_capture/_helpers/text.rs similarity index 64% rename from crates/sqlbuild-python/src/bindings/_helpers/output_capture/text.rs rename to crates/sqlbuild-python-output/src/output_capture/_helpers/text.rs index e36fb753f7..145b915c0e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/text.rs +++ b/crates/sqlbuild-python-output/src/output_capture/_helpers/text.rs @@ -1,17 +1,17 @@ //! `chunk_text` for command-output records. -use pyo3::prelude::{ - Bound, PyAny, PyModule, PyModuleMethods, PyResult, Python, pyfunction, wrap_pyfunction, -}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python, pyfunction}; use pyo3::types::{PyString, PyTuple}; use sqlbuild_core::command_output::main::chunk_output_text::chunk_output_text; -use crate::bindings::_helpers::output_capture::values::{pass_text, python_text, unencodable}; +use crate::output_capture::_helpers::values::pass_text; +use crate::output_capture::_helpers::values::python_text; +use crate::output_capture::_helpers::values::unencodable; /// Split text deterministically without splitting Unicode code points. #[pyfunction] #[pyo3(signature = (*, text, max_bytes))] -fn chunk_command_output_text<'py>( +pub(crate) fn chunk_command_output_text<'py>( py: Python<'py>, text: &Bound<'py, PyString>, max_bytes: i64, @@ -25,7 +25,3 @@ fn chunk_command_output_text<'py>( } PyTuple::new(py, items) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(chunk_command_output_text, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs b/crates/sqlbuild-python-output/src/output_capture/_helpers/values.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs rename to crates/sqlbuild-python-output/src/output_capture/_helpers/values.rs diff --git a/crates/sqlbuild-python-output/src/output_capture/main/invocation_context.rs b/crates/sqlbuild-python-output/src/output_capture/main/invocation_context.rs new file mode 100644 index 0000000000..2887fb9d37 --- /dev/null +++ b/crates/sqlbuild-python-output/src/output_capture/main/invocation_context.rs @@ -0,0 +1,59 @@ +//! Validated opaque invocation metadata supplied by an integration through the environment. + +use crate::output_capture::_helpers::records::input_error; +use crate::output_capture::_helpers::records::validate_depth; +use crate::output_capture::_helpers::values::freeze; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_core::command_output::constants::{ + INVOCATION_CONTEXT_ENV, MAX_INVOCATION_CONTEXT_BYTES, +}; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; + +/// Validated opaque invocation metadata supplied by an integration through the environment. +pub fn invocation_context<'py>( + py: Python<'py>, + environment: Option>, +) -> PyResult> { + let environment = match environment { + Some(environment) if !environment.is_none() => environment, + _ => imported(py, "os", "environ")?, + }; + let raw = mapping_get(&environment, INVOCATION_CONTEXT_ENV)?; + if raw.is_none() { + return Ok(PyDict::new(py).into_any()); + } + let text: String = raw.extract()?; + if text.len() > MAX_INVOCATION_CONTEXT_BYTES { + return Err(input_error( + py, + &format!( + "{INVOCATION_CONTEXT_ENV} must not exceed {MAX_INVOCATION_CONTEXT_BYTES} bytes" + ), + )); + } + let loaded = imported(py, "json", "loads")? + .call1((text,)) + .map_err(|error| match error.is_instance_of::(py) { + true => input_error( + py, + &format!("{INVOCATION_CONTEXT_ENV} must contain valid JSON"), + ), + false => error, + })?; + if !loaded.is_instance_of::() { + return Err(input_error( + py, + &format!("{INVOCATION_CONTEXT_ENV} must contain a JSON object"), + )); + } + validate_depth(&loaded, 1)?; + freeze(&loaded, "external_context").map_err(|error| { + match error.is_instance_of::(py) { + true => input_error(py, &error.value(py).to_string()), + false => error, + } + }) +} diff --git a/crates/sqlbuild-python-output/src/output_capture/main/mod.rs b/crates/sqlbuild-python-output/src/output_capture/main/mod.rs new file mode 100644 index 0000000000..7e4f386c68 --- /dev/null +++ b/crates/sqlbuild-python-output/src/output_capture/main/mod.rs @@ -0,0 +1,5 @@ +pub mod invocation_context; +pub mod register_context; +pub mod register_dispatcher; +pub mod register_records; +pub mod register_text; diff --git a/crates/sqlbuild-python-output/src/output_capture/main/register_context.rs b/crates/sqlbuild-python-output/src/output_capture/main/register_context.rs new file mode 100644 index 0000000000..9c0bfedf1e --- /dev/null +++ b/crates/sqlbuild-python-output/src/output_capture/main/register_context.rs @@ -0,0 +1,13 @@ +//! Register the context functions and classes. + +use crate::output_capture::_helpers::context::CommandOutputContext; +use crate::output_capture::_helpers::context::command_output_context; +use crate::output_capture::_helpers::context::current_output_capture_context; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, wrap_pyfunction}; + +/// Register the context functions and classes. +pub fn register_context(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(command_output_context, module)?)?; + module.add_function(wrap_pyfunction!(current_output_capture_context, module)?) +} diff --git a/crates/sqlbuild-python-output/src/output_capture/main/register_dispatcher.rs b/crates/sqlbuild-python-output/src/output_capture/main/register_dispatcher.rs new file mode 100644 index 0000000000..88c0ec4b7c --- /dev/null +++ b/crates/sqlbuild-python-output/src/output_capture/main/register_dispatcher.rs @@ -0,0 +1,16 @@ +//! Add the capture classes and run `stop_output_capture_threads` at interpreter exit. + +use crate::output_capture::_helpers::dispatcher::OutputCaptureDispatcher; +use crate::output_capture::_helpers::dispatcher::OutputDeliveryWorker; +use crate::output_capture::_helpers::dispatcher::stop_output_capture_threads; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, wrap_pyfunction}; +use sqlbuild_python_base::boundary::main::imported::imported; + +/// Add the capture classes and run `stop_output_capture_threads` at interpreter exit. +pub fn register_dispatcher(module: &Bound<'_, PyModule>) -> PyResult<()> { + let hook = wrap_pyfunction!(stop_output_capture_threads, module)?; + module.add_function(hook.clone())?; + imported(module.py(), "atexit", "register")?.call1((hook,))?; + module.add_class::()?; + module.add_class::() +} diff --git a/crates/sqlbuild-python-output/src/output_capture/main/register_records.rs b/crates/sqlbuild-python-output/src/output_capture/main/register_records.rs new file mode 100644 index 0000000000..92938d518f --- /dev/null +++ b/crates/sqlbuild-python-output/src/output_capture/main/register_records.rs @@ -0,0 +1,18 @@ +//! Register the records functions and classes. + +use crate::output_capture::_helpers::records::command_output_from_json; +use crate::output_capture::_helpers::records::command_output_to_json; +use crate::output_capture::_helpers::records::freeze_command_output_record; +use crate::output_capture::_helpers::records::invocation_context_from_environment; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, wrap_pyfunction}; + +/// Register the records functions and classes. +pub fn register_records(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(freeze_command_output_record, module)?)?; + module.add_function(wrap_pyfunction!(command_output_to_json, module)?)?; + module.add_function(wrap_pyfunction!(command_output_from_json, module)?)?; + module.add_function(wrap_pyfunction!( + invocation_context_from_environment, + module + )?) +} diff --git a/crates/sqlbuild-python-output/src/output_capture/main/register_text.rs b/crates/sqlbuild-python-output/src/output_capture/main/register_text.rs new file mode 100644 index 0000000000..a0dd9d9dff --- /dev/null +++ b/crates/sqlbuild-python-output/src/output_capture/main/register_text.rs @@ -0,0 +1,9 @@ +//! Register the text functions and classes. + +use crate::output_capture::_helpers::text::chunk_command_output_text; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, wrap_pyfunction}; + +/// Register the text functions and classes. +pub fn register_text(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(chunk_command_output_text, module)?) +} diff --git a/crates/sqlbuild-python-output/src/output_capture/mod.rs b/crates/sqlbuild-python-output/src/output_capture/mod.rs new file mode 100644 index 0000000000..a38447a4ab --- /dev/null +++ b/crates/sqlbuild-python-output/src/output_capture/mod.rs @@ -0,0 +1,4 @@ +//! Output capture: capture context, the record dispatcher, invocation records and captured text. + +pub(crate) mod _helpers; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs b/crates/sqlbuild-python-output/src/recall/_helpers/clickhouse_calls.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs rename to crates/sqlbuild-python-output/src/recall/_helpers/clickhouse_calls.rs index 936ab7854a..c2ac8a0cc8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs +++ b/crates/sqlbuild-python-output/src/recall/_helpers/clickhouse_calls.rs @@ -1,5 +1,6 @@ //! Calls into the ClickHouse adapter's Recall store operations, through the native host. +use crate::recall::models::QueryStatistics; use pyo3::prelude::{Bound, FromPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyBytes, PyDict, PyDictMethods, PyInt, PyList, PyString}; use sqlbuild_config::models::ConfigValue; @@ -14,14 +15,6 @@ use sqlbuild_python_base::boundary::main::config_value_to_python::config_value_t use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; -/// What `sqb recall query` reports about a ClickHouse query besides its rows. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub(crate) struct QueryStatistics { - pub(crate) rows_read: Option, - pub(crate) bytes_read: Option, - pub(crate) query_id: Option, -} - /// One result cell as the adapter returns it: plain values only. #[derive(FromPyObject)] enum Cell { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs b/crates/sqlbuild-python-output/src/recall/_helpers/clickhouse_store.rs similarity index 63% rename from crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs rename to crates/sqlbuild-python-output/src/recall/_helpers/clickhouse_store.rs index 5151a20798..ea4d6b4aea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs +++ b/crates/sqlbuild-python-output/src/recall/_helpers/clickhouse_store.rs @@ -1,5 +1,6 @@ //! The ClickHouse Recall store: written and read through the ClickHouse adapter's connection. +use crate::recall::models::ClickHouseLocation; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::{Mutex, PoisonError}; @@ -19,15 +20,13 @@ use sqlbuild_recall::clickhouse::models::{StoredColumn, StoredRow, TtlMismatch}; use sqlbuild_recall::records::main::epoch_micros::epoch_micros; use sqlbuild_recall::records::main::format_timestamp::format_timestamp; use sqlbuild_recall::records::main::meta_row::meta_row; -use sqlbuild_recall::records::models::{OutputStream, RecallRow, RecallValue, RowEnvelope}; +use sqlbuild_recall::records::models::{RecallRow, RecallValue, RowEnvelope}; use sqlbuild_recall::schema::constants::{ ALL_TABLES, ARTIFACT_REFRESH_DAYS, META_SCHEMA_COMPONENT, META_SCHEMA_VERSION_KEY, RECALL_SCHEMA_VERSION, VIEW_NAMES, }; use sqlbuild_recall::schema::main::add_column_statement::add_column_statement; -use sqlbuild_recall::schema::main::clickhouse_relation::clickhouse_relation; use sqlbuild_recall::schema::main::create_database_statement::create_database_statement; -use sqlbuild_recall::schema::main::ddl_statements::ddl_statements; use sqlbuild_recall::schema::main::render_script::render_script; use sqlbuild_recall::schema::main::retention_fix_statements::retention_fix_statements; use sqlbuild_recall::schema::main::schema_age::schema_age; @@ -40,57 +39,18 @@ use sqlbuild_recall::schema::main::version_plan::version_plan; use sqlbuild_recall::schema::main::view_columns::view_columns; use sqlbuild_recall::schema::main::view_statements::view_statements; use sqlbuild_recall::schema::models::{ - ColumnSpec, DdlOptions, Dialect, ObservedSchema, SchemaAge, StoreVersion, Table, VersionPlan, + ColumnSpec, Dialect, ObservedSchema, SchemaAge, StoreVersion, Table, VersionPlan, }; use sqlbuild_recall::settings::constants::CLICKHOUSE_DATABASE_KEY; -use sqlbuild_recall::store::constants::{LAST_INVOCATION_SELECTOR, LISTED_INVOCATION_CANDIDATES}; use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; -use sqlbuild_recall::store::main::chosen_invocation::chosen_invocation; -use sqlbuild_recall::store::models::{ - DescribedColumn, DescribedRelation, InsertOutcome, OutputTranscript, QueryResult, -}; +use sqlbuild_recall::store::models::{InsertOutcome, QueryResult}; use sqlbuild_recall::store::types::{RecallStore, RecallStoreOpener}; -use crate::bindings::_helpers::recall::clickhouse_calls::{ - self as calls, ConnectionRole, QueryStatistics, StoreHandle, -}; -use crate::bindings::_helpers::recall::constants::{ - READ_ONLY_QUERIES, READONLY_SETTING, SECONDS_PER_DAY, SERVER_DEFAULT_DATABASE, - UNKNOWN_TABLE_ERROR, -}; - -/// Where a ClickHouse store lives and how long it keeps history. -#[derive(Clone, Debug)] -pub(crate) struct ClickHouseLocation { - pub(crate) config: Vec<(String, ConfigValue)>, - pub(crate) options: DdlOptions, -} - -impl ClickHouseLocation { - pub(crate) fn new(config: &[(String, ConfigValue)], database: &str, retention: u32) -> Self { - Self { - config: config.to_vec(), - options: DdlOptions { - database: Some(database.to_owned()), - retention_days: retention, - cluster: None, - producer_version: env!("CARGO_PKG_VERSION").to_owned(), - }, - } - } - - fn connect_action(&self) -> String { - format!("connect to the Recall store in `{}`", self.database()) - } - - fn database(&self) -> &str { - self.options.database.as_deref().unwrap_or_default() - } - - fn table(&self, name: &str) -> String { - clickhouse_relation(name, &self.options) - } -} +use crate::recall::_helpers::clickhouse_calls::ConnectionRole; +use crate::recall::_helpers::clickhouse_calls::StoreHandle; +use crate::recall::_helpers::clickhouse_calls::connect; +use crate::recall::constants::SECONDS_PER_DAY; +use crate::recall::constants::SERVER_DEFAULT_DATABASE; /// Opens the store on the delivery thread; `retention_notice` decides whether a retention conflict is reported this time. #[derive(Debug)] @@ -103,32 +63,33 @@ pub(crate) struct ClickHouseStoreOpener { } /// The Recall relations present: tables with their `CREATE` queries, views, and the stored schema version. -struct StoreState { - tables: Vec<(String, String)>, - views: HashSet, +pub(crate) struct StoreState { + pub(crate) tables: Vec<(String, String)>, + pub(crate) views: HashSet, /// The views' `CREATE` queries, which show what an older version wrote when columns cannot. - view_definitions: Vec, - version: i64, + pub(crate) view_definitions: Vec, + pub(crate) version: i64, /// Seconds since the newest version row was recorded. - version_age_seconds: i64, + pub(crate) version_age_seconds: i64, } /// A store ready to write: its tables with their `CREATE` queries, and the upgrade an administrator must run when this user could not. -struct ReadyStore { +pub(crate) struct ReadyStore { tables: Vec<(String, String)>, behind: Vec, } /// The store the delivery thread writes. -struct ClickHouseStore { - location: ClickHouseLocation, - connection: Option, - columns: HashMap>, - notices: Vec, - information: Vec, +#[derive(Debug)] +pub(crate) struct ClickHouseStore { + pub(crate) location: ClickHouseLocation, + pub(crate) connection: Option, + pub(crate) columns: HashMap>, + pub(crate) notices: Vec, + pub(crate) information: Vec, } -fn text(value: Option<&RecallValue>) -> String { +pub(crate) fn text(value: Option<&RecallValue>) -> String { match value { Some(RecallValue::Text(text)) => text.clone(), Some(RecallValue::Blob(data)) => String::from_utf8_lossy(data).into_owned(), @@ -137,7 +98,7 @@ fn text(value: Option<&RecallValue>) -> String { } } -fn integer(value: Option<&RecallValue>) -> i64 { +pub(crate) fn integer(value: Option<&RecallValue>) -> i64 { match value { Some(RecallValue::Integer(number)) => *number, _ => 0, @@ -145,28 +106,44 @@ fn integer(value: Option<&RecallValue>) -> i64 { } /// `text` as a ClickHouse string literal. -fn literal(text: &str) -> String { +pub(crate) fn literal(text: &str) -> String { format!("'{}'", text.replace('\\', "\\\\").replace('\'', "\\'")) } impl ClickHouseStore { - fn connection(&self) -> Result { + pub(crate) fn connection(&self) -> Result { self.connection.ok_or_else(|| StoreError { kind: StoreErrorKind::Other, message: "the Recall store connection is closed".to_owned(), }) } - fn query(&self, py: Python<'_>, sql: &str, action: &str) -> Result { - calls::query(py, self.connection()?, (sql, &[], None), action).map(|(result, _)| result) + pub(crate) fn query( + &self, + py: Python<'_>, + sql: &str, + action: &str, + ) -> Result { + crate::recall::_helpers::clickhouse_calls::query( + py, + self.connection()?, + (sql, &[], None), + action, + ) + .map(|(result, _)| result) } - fn command(&self, py: Python<'_>, sql: &str, action: &str) -> Result<(), StoreError> { - calls::command(py, self.connection()?, sql, action) + pub(crate) fn command( + &self, + py: Python<'_>, + sql: &str, + action: &str, + ) -> Result<(), StoreError> { + crate::recall::_helpers::clickhouse_calls::command(py, self.connection()?, sql, action) } /// The Recall relations present, with their `CREATE` queries, and the stored schema version. - fn state(&self, py: Python<'_>) -> Result { + pub(crate) fn state(&self, py: Python<'_>) -> Result { let sql = store_state_sql(self.location.database()); let relations: QueryResult = self.query(py, &sql.relations, "inspect the Recall store")?; let mut tables: Vec<(String, String)> = Vec::new(); @@ -204,7 +181,7 @@ impl ClickHouseStore { } /// The recorded version, or without one how the tables, read by `read_columns`, and the views compare with this build's; either is lowered when a view definition shows an older version wrote it. - fn store_version( + pub(crate) fn store_version( &self, py: Python<'_>, state: &StoreState, @@ -216,7 +193,7 @@ impl ClickHouseStore { )) } - fn columns_version( + pub(crate) fn columns_version( &self, py: Python<'_>, state: &StoreState, @@ -263,7 +240,7 @@ impl ClickHouseStore { ))) } - fn database_exists(&self, py: Python<'_>) -> Result { + pub(crate) fn database_exists(&self, py: Python<'_>) -> Result { let found = self.query( py, &format!( @@ -276,7 +253,7 @@ impl ClickHouseStore { } /// Create the configured database when it is missing, saying so; an existing one, whatever its engine, is left alone. A user who may not create it gets the statement to hand to an administrator. - fn ensure_database(&mut self, py: Python<'_>) -> Result<(), StoreError> { + pub(crate) fn ensure_database(&mut self, py: Python<'_>) -> Result<(), StoreError> { if self.database_exists(py)? { return Ok(()); } @@ -303,7 +280,7 @@ impl ClickHouseStore { } } - fn read_columns(&mut self, py: Python<'_>) -> Result<(), StoreError> { + pub(crate) fn read_columns(&mut self, py: Python<'_>) -> Result<(), StoreError> { let sql = store_state_sql(self.location.database()); let found = self.query(py, &sql.columns, "inspect the Recall tables")?; let mut columns: HashMap> = HashMap::new(); @@ -335,7 +312,7 @@ impl ClickHouseStore { } /// What the store holds, from its read columns, `views` and recorded `version`. - fn observed(&self, views: &HashSet, version: i64) -> ObservedSchema { + pub(crate) fn observed(&self, views: &HashSet, version: i64) -> ObservedSchema { let mut observed = ObservedSchema { views: views.iter().cloned().collect(), version: (version > 0).then_some(version), @@ -351,7 +328,7 @@ impl ClickHouseStore { } /// Create a store whose tables are missing. On existing tables add missing columns, create missing views, and replace views on a version upgrade; when retention removed the version row, the views' columns tell whether the store is older, current or newer, and a newer store's views are never replaced; views whose definitions show an older version wrote them make the store older either way. A user who may not run that DDL writes the columns that exist and gets the statements an administrator must run. A version is recorded only once the views are settled, never lower than the store holds, and again before retention removes the newest row. - fn ensure_schema( + pub(crate) fn ensure_schema( &mut self, py: Python<'_>, envelope: &RowEnvelope, @@ -412,7 +389,7 @@ impl ClickHouseStore { } /// Create every table, column and view of a store whose tables are missing. - fn create( + pub(crate) fn create( &mut self, py: Python<'_>, envelope: &RowEnvelope, @@ -438,7 +415,7 @@ impl ClickHouseStore { } /// Add the additive columns the stored tables lack; `true` when the user may not. - fn add_columns(&mut self, py: Python<'_>) -> Result { + pub(crate) fn add_columns(&mut self, py: Python<'_>) -> Result { let missing: Vec<(Table, ColumnSpec)> = self.missing_columns(); let statements: Vec = missing .iter() @@ -467,7 +444,7 @@ impl ClickHouseStore { } /// Every column this build writes that the stored tables lack. - fn missing_columns(&self) -> Vec<(Table, ColumnSpec)> { + pub(crate) fn missing_columns(&self) -> Vec<(Table, ColumnSpec)> { let mut missing: Vec<(Table, ColumnSpec)> = Vec::new(); for table in ALL_TABLES { let present: &[StoredColumn] = self.columns.get(&table).map_or(&[], Vec::as_slice); @@ -481,7 +458,7 @@ impl ClickHouseStore { } /// Record this build's schema version with an insert, which a writer allowed only `SELECT` and `INSERT` may run. - fn record_version( + pub(crate) fn record_version( &self, py: Python<'_>, envelope: &RowEnvelope, @@ -500,7 +477,7 @@ impl ClickHouseStore { self.write(py, Table::Meta, &[row], SystemTime::now()) } - fn write( + pub(crate) fn write( &self, py: Python<'_>, table: Table, @@ -513,7 +490,7 @@ impl ClickHouseStore { kind: StoreErrorKind::Other, message, })?; - calls::insert( + crate::recall::_helpers::clickhouse_calls::insert( py, self.connection()?, (&self.location.table(table_spec(table).name), &names, &block), @@ -521,7 +498,7 @@ impl ClickHouseStore { ) } - fn stored_rows( + pub(crate) fn stored_rows( &self, py: Python<'_>, rows: &[RecallRow], @@ -548,7 +525,7 @@ impl ClickHouseStore { } /// The notice for a store behind this build that this user may not upgrade: history is written to the columns that exist until an administrator runs `statements`. - fn behind_notice(&self, statements: &[String]) -> String { + pub(crate) fn behind_notice(&self, statements: &[String]) -> String { format!( "the Recall store in ClickHouse database `{}` is behind this SQLBuild version, and \ this user may not upgrade it; history is still written, without what the newer \ @@ -559,7 +536,7 @@ impl ClickHouseStore { ) } - fn retention_conflict(&self, tables: &[(String, String)]) -> Option { + pub(crate) fn retention_conflict(&self, tables: &[(String, String)]) -> Option { let mismatches: Vec = ttl_mismatches(tables, self.location.options.retention_days); let first: &TtlMismatch = mismatches.first()?; @@ -623,7 +600,7 @@ impl RecallStore for ClickHouseStore { fn close(&mut self) { if let Some(connection) = self.connection.take() { - Python::attach(|py| calls::close(py, connection)); + Python::attach(|py| crate::recall::_helpers::clickhouse_calls::close(py, connection)); } } } @@ -693,299 +670,18 @@ impl RecallStoreOpener for ClickHouseStoreOpener { } } -/// A ClickHouse store opened for `sqb recall`. -pub(crate) struct ClickHouseReader { - store: ClickHouseStore, -} - -impl ClickHouseReader { - pub(crate) fn open(py: Python<'_>, location: &ClickHouseLocation) -> Result { - let store = ClickHouseStore { - location: location.clone(), - connection: Some(calls::connect( - py, - (&location.config, ConnectionRole::Reader), - &location.connect_action(), - )?), - columns: HashMap::new(), - notices: Vec::new(), - information: Vec::new(), - }; - Ok(Self { store }) - } - - /// Fail with "no history yet" when the store's tables or views are missing; queries check only after they fail, so a successful one costs no extra round trip. - pub(crate) fn require_store(&self, py: Python<'_>) -> Result<(), StoreError> { - let state: StoreState = self.store.state(py)?; - if state.tables.len() < ALL_TABLES.len() || state.views.len() < VIEW_NAMES.len() { - return Err(StoreError { - kind: StoreErrorKind::NotFound, - message: self.not_created_message(), - }); - } - Ok(()) - } - - fn not_created_message(&self) -> String { - format!( - "no Recall history is recorded yet in ClickHouse database `{}`, or its tables are \ - still being created; run a command first", - self.database() - ) - } - - /// The statements that create the store in `location`, or bring it up to this build's schema, read with `SELECT` only. - pub(crate) fn upgrade_statements( - py: Python<'_>, - location: &ClickHouseLocation, - ) -> Result, StoreError> { - let mut store = ClickHouseStore { - location: location.clone(), - connection: Some(connect_store(py, location, ConnectionRole::Reader)?), - columns: HashMap::new(), - notices: Vec::new(), - information: Vec::new(), - }; - match store.database_exists(py) { - Ok(true) => {} - Ok(false) => { - store.close(); - return Ok(create_database_statement(&location.options) - .into_iter() - .chain(ddl_statements(Dialect::ClickHouse, &location.options)) - .collect()); - } - Err(error) => { - store.close(); - return Err(error); - } - } - let observed = { - let state: StoreState = store.state(py)?; - store.read_columns(py)?; - let version: i64 = if state.tables.len() < ALL_TABLES.len() { - state.version - } else { - observed_version(store.store_version(py, &state)?) - }; - Ok(store.observed(&state.views, version)) - }; - store.close(); - Ok(upgrade_statements( - Dialect::ClickHouse, - &location.options, - &observed?, - )) - } - - /// The database the store lives in. - pub(crate) fn database(&self) -> &str { - self.store.location.database() - } - - /// At most `limit` rows of a read-only query, with the server's statistics. A failure on a store that is not created yet is a store failure (C971), not an invalid query. An unknown relation on a complete store is retried once, since its views may have been created since the query ran; a relation still unknown is the query's own error (C974). - pub(crate) fn query( - &self, - py: Python<'_>, - sql: &str, - limit: Option, - ) -> Result<(QueryResult, QueryStatistics), StoreError> { - let settings: [(&str, i64); 1] = [(READONLY_SETTING, READ_ONLY_QUERIES)]; - let row_cap: Option = limit.map(|limit| limit.saturating_add(1)); - let run = || { - calls::query( - py, - self.store.connection()?, - (sql, &settings, row_cap), - "run the Recall query", - ) - }; - let rejected = |failure: StoreError| StoreError { - kind: if failure.kind == StoreErrorKind::Busy { - StoreErrorKind::Other - } else { - StoreErrorKind::Query - }, - message: failure.message, - }; - let (mut result, statistics) = match run() { - Ok(found) => found, - Err(failure) => match self.require_store(py) { - Err(missing) if missing.kind == StoreErrorKind::NotFound => { - return Err(StoreError { - kind: StoreErrorKind::Other, - message: missing.message, - }); - } - Ok(()) if failure.message.contains(UNKNOWN_TABLE_ERROR) => { - run().map_err(rejected)? - } - Ok(()) | Err(_) => return Err(rejected(failure)), - }, - }; - if let Some(limit) = limit - && result.rows.len() > limit - { - result.rows.truncate(limit); - result.truncated = true; - } - Ok((result, statistics)) - } - - /// The store's views, then its tables, with their columns. - pub(crate) fn describe(&self, py: Python<'_>) -> Result, StoreError> { - let names: Vec = VIEW_NAMES - .iter() - .map(|name| literal(name)) - .chain( - ALL_TABLES - .iter() - .map(|table| literal(table_spec(*table).name)), - ) - .collect(); - let found = self.store.query( - py, - &format!( - "SELECT c.table, if(t.engine = 'View', 'view', 'table') AS kind, c.name, c.type \ - FROM system.columns AS c INNER JOIN system.tables AS t \ - ON t.database = c.database AND t.name = c.table \ - WHERE c.database = {} AND c.table IN ({}) \ - ORDER BY kind = 'table', c.table, c.position", - literal(self.database()), - names.join(", ") - ), - "describe the Recall store", - )?; - let mut relations: Vec = Vec::new(); - for row in &found.rows { - let name = text(row.first()); - if relations.last().is_none_or(|last| last.name != name) { - relations.push(DescribedRelation { - name, - kind: text(row.get(1)), - columns: Vec::new(), - }); - } - let declared_type = text(row.get(3)); - if let Some(relation) = relations.last_mut() { - relation.columns.push(DescribedColumn { - name: text(row.get(2)), - nullable: declared_type.starts_with("Nullable("), - declared_type, - }); - } - } - Ok(relations) - } - - /// The full invocation ID an ID, unique prefix or `last` names. - pub(crate) fn resolve_invocation( - &self, - py: Python<'_>, - selector: &str, - ) -> Result { - let location = &self.store.location; - let started = format!( - "(SELECT invocation_id, invocation_started_at FROM {} \ - UNION ALL SELECT invocation_id, invocation_started_at FROM {})", - location.table(table_spec(Table::Invocations).name), - location.table(table_spec(Table::Heartbeats).name), - ); - let sql = if selector == LAST_INVOCATION_SELECTOR { - format!( - "SELECT invocation_id FROM {started} \ - ORDER BY invocation_started_at DESC, invocation_id DESC LIMIT 1" - ) - } else { - format!( - "SELECT DISTINCT invocation_id FROM {started} \ - WHERE startsWith(invocation_id, {}) ORDER BY invocation_id LIMIT {}", - literal(selector), - LISTED_INVOCATION_CANDIDATES + 1 - ) - }; - let found = self.store.query(py, &sql, "find the invocation")?; - let candidates: Vec = found.rows.iter().map(|row| text(row.first())).collect(); - chosen_invocation(selector, &candidates) - } - - /// One invocation's exact output bytes, deduplicated by sequence, and its completeness. - pub(crate) fn read_output_transcript( - &self, - py: Python<'_>, - invocation_id: &str, - stream: Option, - ) -> Result { - let id = literal(invocation_id); - let stream_filter: String = stream.map_or_else(String::new, |stream| { - format!(" AND stream = {}", literal(stream.as_str())) - }); - let output = self.store.location.table(table_spec(Table::Output).name); - let chunks = self.store.query( - py, - &format!( - "SELECT data FROM (SELECT sequence, stream, data FROM {output} \ - WHERE invocation_id = {id} ORDER BY sequence, ingested_at LIMIT 1 BY sequence) \ - WHERE 1{stream_filter} ORDER BY sequence" - ), - "read Recall output", - )?; - let mut transcript = OutputTranscript { - invocation_id: invocation_id.to_owned(), - ..OutputTranscript::default() - }; - for row in &chunks.rows { - match row.first() { - Some(RecallValue::Blob(data)) => transcript.data.extend_from_slice(data), - Some(RecallValue::Text(text)) => transcript.data.extend_from_slice(text.as_bytes()), - _ => {} - } - } - let database = self.store.location.table("invocations"); - let status = self.store.query( - py, - &format!("SELECT status FROM {database} WHERE invocation_id = {id}"), - "read the Recall invocation", - )?; - transcript.status = status.rows.first().map(|row| text(row.first())); - let completeness_view = self.store.location.table("output_invocations"); - let completeness = self.store.query( - py, - &format!( - "SELECT missing_sequences, reported_drops, filtered, flush_complete, complete, \ - gap_reason FROM {completeness_view} WHERE invocation_id = {id}" - ), - "read Recall output completeness", - )?; - if let Some(row) = completeness.rows.first() { - transcript.missing_sequences = integer(row.first()); - transcript.reported_drops = integer(row.get(1)); - transcript.filtered = integer(row.get(2)); - transcript.flush_complete = integer(row.get(3)) != 0; - transcript.complete = integer(row.get(4)) != 0; - let reason = text(row.get(5)); - transcript.gap_reason = (!reason.is_empty()).then_some(reason); - } - Ok(transcript) - } - - pub(crate) fn close(mut self) { - self.store.close(); - } -} - /// Why a store cannot be created when its database is missing and the writer may not create it. pub(crate) fn missing_database_reason(database: &str) -> String { format!("ClickHouse database `{database}` does not exist, and this user may not create it") } /// Connect for `role`. When the configured database does not exist yet, connect without it, so that it can be created or reported; every statement names the database itself. -fn connect_store( +pub(crate) fn connect_store( py: Python<'_>, location: &ClickHouseLocation, role: ConnectionRole, ) -> Result { - match calls::connect(py, (&location.config, role), &location.connect_action()) { + match connect(py, (&location.config, role), &location.connect_action()) { Err(error) if error.kind == StoreErrorKind::NotFound => { let config: Vec<(String, ConfigValue)> = location .config @@ -1001,14 +697,14 @@ fn connect_store( } }) .collect(); - calls::connect(py, (&config, role), &location.connect_action()) + connect(py, (&config, role), &location.connect_action()) } connected => connected, } } /// The version `upgrade_statements` compares with: a store without a version row whose views are current or newer needs no view replacement. -fn observed_version(store: StoreVersion) -> i64 { +pub(crate) fn observed_version(store: StoreVersion) -> i64 { match store { StoreVersion::Recorded { version, .. } => version, StoreVersion::Unrecorded(SchemaAge::Older) => 0, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs b/crates/sqlbuild-python-output/src/recall/_helpers/events.rs similarity index 87% rename from crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs rename to crates/sqlbuild-python-output/src/recall/_helpers/events.rs index f7960e287c..07153fcb1a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs +++ b/crates/sqlbuild-python-output/src/recall/_helpers/events.rs @@ -1,30 +1,23 @@ //! Lifecycle events as the JSON object `lifecycle_event_to_json` encodes, built natively. +use crate::recall::main::escaped_text::escaped_text; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyMapping, PyString, - PyStringMethods, PyTuple, + PyTuple, }; use serde_json::{Map, Number, Value}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::recall::constants::{ - EVENT_OPTIONAL_TEXT_FIELDS, EVENT_TEXT_FIELDS, INVOCATION_METADATA_SCHEMA_VERSION, - TIMESTAMP_PARTS, UTC_ZONE_NAME, -}; +use crate::recall::constants::EVENT_OPTIONAL_TEXT_FIELDS; +use crate::recall::constants::EVENT_TEXT_FIELDS; +use crate::recall::constants::INVOCATION_METADATA_SCHEMA_VERSION; +use crate::recall::constants::TIMESTAMP_PARTS; +use crate::recall::constants::UTC_ZONE_NAME; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::discovery::main::snapshots::escaped::escaped; - -/// The text of `text`; lone surrogates, which UTF-8 cannot hold, are kept as `\\xNN` or `\\uXXXX` escapes. -pub(crate) fn escaped_text(text: &Bound<'_, PyString>) -> PyResult { - match text.to_str() { - Ok(valid) => Ok(valid.to_owned()), - Err(_) => escaped(text), - } -} -fn text(value: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn text(value: &Bound<'_, PyAny>) -> PyResult> { if value.is_none() { return Ok(None); } diff --git a/crates/sqlbuild-python-output/src/recall/_helpers/mod.rs b/crates/sqlbuild-python-output/src/recall/_helpers/mod.rs new file mode 100644 index 0000000000..91283f0f65 --- /dev/null +++ b/crates/sqlbuild-python-output/src/recall/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod clickhouse_calls; +pub(crate) mod clickhouse_store; +pub(crate) mod events; +pub(crate) mod replacements; +pub(crate) mod session; +pub(crate) mod sinks; +pub(crate) mod writer; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs b/crates/sqlbuild-python-output/src/recall/_helpers/replacements.rs similarity index 56% rename from crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs rename to crates/sqlbuild-python-output/src/recall/_helpers/replacements.rs index 657ffa9d37..1da5707f8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs +++ b/crates/sqlbuild-python-output/src/recall/_helpers/replacements.rs @@ -1,34 +1,29 @@ //! Recall replacement sinks: the one lifecycle and one command-output sink that decide what Recall stores. use std::cell::Cell; -use std::sync::{Arc, OnceLock}; +use std::sync::Arc; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyString}; use sqlbuild_core::command_output::constants::RECALL_REPLACEMENT_SCHEMA_VERSION; use sqlbuild_core::command_output::main::ansi_free_text::ansi_free_text; -use sqlbuild_core::event_export::main::route_lifecycle_event::route_lifecycle_event; -use sqlbuild_core::event_export::models::{ExporterCounts, ExporterFilter}; +use sqlbuild_core::event_export::models::ExporterFilter; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; use sqlbuild_recall::records::models::{OutputChunk, OutputStream}; -use sqlbuild_recall::writer::models::{ - DeliveryStream, RecallSession, ReplacedEvents, ReplacedOutput, -}; +use sqlbuild_recall::writer::models::{ReplacedEvents, ReplacedOutput}; use sqlbuild_recall::writer::types::{OutputReplacement, PendingLifecycleEvent}; -use crate::bindings::_helpers::output_capture::records::invocation_context; -use crate::bindings::_helpers::recall::constants::{ - EVENT_PARAMETER, RECORD_PARAMETER, UTC_OFFSET, UTC_SUFFIX, -}; -use crate::bindings::_helpers::recall::writer::RecallWriter; +use crate::output_capture::main::invocation_context::invocation_context; +use crate::recall::constants::EVENT_PARAMETER; +use crate::recall::constants::RECORD_PARAMETER; +use crate::recall::constants::UTC_OFFSET; +use crate::recall::constants::UTC_SUFFIX; +use crate::recall::models::RecallWriter; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::event_exporting::main::export_policy::export_policy; -use sqlbuild_python_base::event_exporting::models::BoundEventExporter; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; -use sqlbuild_python_base::records::main::fields::field::field as native_field; thread_local! { /// Set while this thread runs a replacement sink, whose own writes Recall never records. @@ -58,25 +53,10 @@ fn as_replacement_sink(call: impl FnOnce(Python<'_>) -> PyResult, failed: /// The lifecycle replacement: the sink function, the parameter its writer goes in, and its event filter. #[derive(Debug)] -struct LifecycleReplacement { - function: Py, - writer_parameter: String, - filter: ExporterFilter, -} - -/// The replacement sinks of one invocation, configured once discovery has run. -#[derive(Default)] -pub(crate) struct RecallReplacements { - lifecycle: OnceLock>, -} - -impl std::fmt::Debug for RecallReplacements { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("RecallReplacements") - .field("lifecycle", &self.lifecycle.get().is_some()) - .finish() - } +pub(crate) struct LifecycleReplacement { + pub(crate) function: Py, + pub(crate) writer_parameter: String, + pub(crate) filter: ExporterFilter, } /// `function(**{parameter: record, writer_parameter: writer})` through the user-code shim; whether it raised. @@ -100,7 +80,9 @@ fn invoke( } /// The `(sink, writer_parameter)` pair discovery found, or `None`. -fn discovered<'py>(pair: &Bound<'py, PyAny>) -> PyResult, String)>> { +pub(crate) fn discovered<'py>( + pair: &Bound<'py, PyAny>, +) -> PyResult, String)>> { if pair.is_none() { return Ok(None); } @@ -108,96 +90,15 @@ fn discovered<'py>(pair: &Bound<'py, PyAny>) -> PyResult, - (lifecycle, output): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - ) -> PyResult { - let mut active: bool = false; - if let Some((sink, writer_parameter)) = discovered(lifecycle)? { - let bound = BoundEventExporter::create( - native_field(&sink, "name")?.extract()?, - ( - native_field(&sink, "function")?.unbind(), - PyDict::new(sink.py()).into_any().unbind(), - ), - ( - Some(&native_field(&sink, "event_kinds")?), - Some(&native_field(&sink, "min_severity")?), - ), - )?; - drop(self.lifecycle.set(Arc::new(LifecycleReplacement { - function: bound.function, - writer_parameter, - filter: bound.filter, - }))); - active = true; - } - if let Some((sink, writer_parameter)) = discovered(output)? { - let mut streams: [bool; 2] = [false; 2]; - for stream in native_field(&sink, "streams")?.try_iter()? { - let name: String = native_field(&stream?, "value")?.extract()?; - if let Some(stream) = OutputStream::from_name(&name) { - streams[usize::from(stream == OutputStream::Stderr)] = true; - } - } - session.set_output_replacement(Arc::new(OutputSink { - function: native_field(&sink, "function")?.unbind(), - writer_parameter, - streams, - invocation_id: session.invocation_id(), - })); - active = true; - } - Ok(active) - } - - /// Queue `event` for the lifecycle replacement, which runs on the delivery thread; `false` when there is none. - pub(crate) fn deliver_lifecycle( - &self, - py: Python<'_>, - session: &Arc, - event: &Bound<'_, PyAny>, - ) -> PyResult { - let Some(replacement) = self.lifecycle.get() else { - return Ok(false); - }; - let event_type: String = native_field(event, "event_type")?.extract()?; - let eligible: bool = export_policy(py, &event_type).is_ok_and(|policy| { - let (_, eligible) = route_lifecycle_event( - &policy, - std::slice::from_ref(&replacement.filter), - vec![ExporterCounts::default()], - ); - !eligible.is_empty() - }); - if !eligible { - session.record_filtered(DeliveryStream::Events, 1); - return Ok(true); - } - let schema_version: i64 = native_field(event, "schema_version")?.extract()?; - session.publish_pending_event( - Box::new(PendingEvent { - replacement: Arc::clone(replacement), - event: event.clone().unbind(), - }), - (&event_type, schema_version), - ); - Ok(true) - } -} - /// One lifecycle event queued for the lifecycle replacement. #[derive(Debug)] -struct PendingEvent { - replacement: Arc, - event: Py, +pub(crate) struct PendingEvent { + pub(crate) replacement: Arc, + pub(crate) event: Py, } impl PendingEvent { - fn call(&self, py: Python<'_>) -> PyResult { + pub(crate) fn call(&self, py: Python<'_>) -> PyResult { let writer = Bound::new(py, RecallWriter::lifecycle())?; let raised: bool = invoke( py, @@ -224,12 +125,12 @@ impl PendingLifecycleEvent for PendingEvent { } /// The output replacement, called on the Recall delivery thread for each chunk. -struct OutputSink { - function: Py, - writer_parameter: String, +pub(crate) struct OutputSink { + pub(crate) function: Py, + pub(crate) writer_parameter: String, /// Whether the sink takes stdout and stderr chunks; others are filtered. - streams: [bool; 2], - invocation_id: String, + pub(crate) streams: [bool; 2], + pub(crate) invocation_id: String, } impl std::fmt::Debug for OutputSink { @@ -244,7 +145,11 @@ impl std::fmt::Debug for OutputSink { impl OutputSink { /// The schema-2 `CommandOutputRecord` of `chunk`. - fn record<'py>(&self, py: Python<'py>, chunk: &OutputChunk) -> PyResult> { + pub(crate) fn record<'py>( + &self, + py: Python<'py>, + chunk: &OutputChunk, + ) -> PyResult> { let occurred_at: Bound<'py, PyAny> = imported(py, "datetime", "datetime")?.call_method1( "fromisoformat", (chunk.occurred_at.replace(UTC_SUFFIX, UTC_OFFSET),), @@ -279,7 +184,7 @@ impl OutputSink { ) } - fn call(&self, py: Python<'_>, chunk: &OutputChunk) -> PyResult { + pub(crate) fn call(&self, py: Python<'_>, chunk: &OutputChunk) -> PyResult { let record = self.record(py, chunk)?; let writer = Bound::new(py, RecallWriter::output(chunk.sequence))?; let raised: bool = invoke( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python-output/src/recall/_helpers/session.rs similarity index 67% rename from crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs rename to crates/sqlbuild-python-output/src/recall/_helpers/session.rs index e87eda9f4e..0a69f989fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python-output/src/recall/_helpers/session.rs @@ -1,5 +1,9 @@ //! The invocation's Recall session: started before the command runs and closed after it returns. +use crate::recall::main::effective_selection::effective_selection; +use crate::recall::main::escaped_text::escaped_text; +use crate::recall::main::write_stderr::write_stderr; +use crate::recall::models::InvocationRecall; use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; @@ -7,8 +11,6 @@ use std::time::{Duration, Instant, SystemTime}; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use sqlbuild_cli::dispatch::main::closing_status::closing_status; -use sqlbuild_cli::dispatch::main::selector_file_selectors::selector_file_selectors; use sqlbuild_cli::dispatch::models::CommandFacts; use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; use sqlbuild_host::entries::models::ExecutorHost; @@ -17,15 +19,12 @@ use sqlbuild_recall::records::main::command_mode::command_mode; use sqlbuild_recall::records::main::format_timestamp::format_timestamp; use sqlbuild_recall::records::main::invocation_origin::invocation_origin; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; -use sqlbuild_recall::records::models::{InvocationContext, OutputStream, RowEnvelope}; +use sqlbuild_recall::records::models::{InvocationContext, RowEnvelope}; use sqlbuild_recall::schema::main::create_database_statement::create_database_statement; use sqlbuild_recall::schema::main::ddl_script::ddl_script; use sqlbuild_recall::schema::main::render_script::render_script; use sqlbuild_recall::schema::models::{DdlOptions, Dialect}; -use sqlbuild_recall::settings::main::resolve_recall_settings::resolve_recall_settings; -use sqlbuild_recall::settings::models::{ - AuditEvidence, RecallSettings, SettingsRequest, StoreLocation, StorePurpose, -}; +use sqlbuild_recall::settings::models::{RecallSettings, StoreLocation}; use sqlbuild_recall::store::main::observe_store_file::observe_store_file; use sqlbuild_recall::store::main::quarantine_store::quarantine_store; use sqlbuild_recall::store::main::store_location_problem::store_location_problem; @@ -36,48 +35,25 @@ use sqlbuild_recall::writer::models::{ DeliverySummary, RecallSession, SessionConfig, SessionStore, SetupOutcome, }; -use crate::bindings::_helpers::output_capture::records::invocation_context; -use crate::bindings::_helpers::recall::clickhouse_store::{ - ClickHouseLocation, ClickHouseStoreOpener, missing_database_reason, -}; -use crate::bindings::_helpers::recall::console::write_stderr; -use crate::bindings::_helpers::recall::constants::{ - CLOSE_POLL_MS, DDL_NOTICE_PREFIX, DEBUG_COMMAND, GIT_DIRTY_ENVIRONMENT, - GIT_REVISION_ENVIRONMENT, HEARTBEAT_ENVIRONMENT, HOST_TOKEN_LENGTH, MINIMUM_HEARTBEAT_SECONDS, - NO_CONNECTION_ATTRIBUTE, NOTICE_INTERVAL_SECONDS, QUARANTINE_START_WAIT_MS, RECALL_HISTORY, - STDOUT_STREAM_INDEX, -}; -use crate::bindings::_helpers::recall::events::escaped_text; -use crate::bindings::_helpers::recall::replacements::{RecallReplacements, in_replacement_sink}; -use crate::bindings::_helpers::recall::sinks::InvocationEvents; +use crate::output_capture::main::invocation_context::invocation_context; +use crate::recall::_helpers::clickhouse_store::ClickHouseStoreOpener; +use crate::recall::_helpers::clickhouse_store::missing_database_reason; +use crate::recall::constants::DDL_NOTICE_PREFIX; +use crate::recall::constants::DEBUG_COMMAND; +use crate::recall::constants::GIT_DIRTY_ENVIRONMENT; +use crate::recall::constants::GIT_REVISION_ENVIRONMENT; +use crate::recall::constants::HEARTBEAT_ENVIRONMENT; +use crate::recall::constants::HOST_TOKEN_LENGTH; +use crate::recall::constants::MINIMUM_HEARTBEAT_SECONDS; +use crate::recall::constants::NO_CONNECTION_ATTRIBUTE; +use crate::recall::constants::NOTICE_INTERVAL_SECONDS; +use crate::recall::constants::QUARANTINE_START_WAIT_MS; +use crate::recall::models::ClickHouseLocation; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_project::project_snapshot::models::WorkerBuild; - -/// What the invocation records: its session, when enabled, and the resolved target and adapter. -#[derive(Clone, Debug, Default)] -pub(crate) struct InvocationRecall { - /// The verified worker remains owned until Recall has drained and its claim can be released. - pub(crate) worker: Arc>>>, - pub(crate) session: Option>, - pub(crate) target: Option, - pub(crate) adapter: Option, - /// The command name, for the closing status line. - pub(crate) command: String, - /// The resolved settings and project directory, for notices at the end of the run. - pub(crate) settings: Option<(RecallSettings, PathBuf)>, - /// The Recall replacement sinks, configured once runtime extensions are discovered. - pub(crate) replacements: Arc, - /// A Recall configuration mistake the command fails with before it runs. - pub(crate) configuration_error: Option, - /// The invocation's own lifecycle events the session has recorded. - pub(crate) invocation_events: Arc, - /// Which failing audits collect evidence rows, as `recall.audit_evidence` sets it. - pub(crate) audit_evidence: AuditEvidence, -} /// The text of an optional attribute value. -fn text_value(value: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn text_value(value: &Bound<'_, PyAny>) -> PyResult> { if value.is_none() { return Ok(None); } @@ -88,7 +64,7 @@ fn attribute<'py>(args: &Bound<'py, PyAny>, name: &str) -> PyResult, name: &str) -> PyResult> { +pub(crate) fn list_attribute(args: &Bound<'_, PyAny>, name: &str) -> PyResult> { let value: Bound<'_, PyAny> = attribute(args, name)?; let mut items: Vec = Vec::new(); if value.is_none() { @@ -100,30 +76,12 @@ fn list_attribute(args: &Bound<'_, PyAny>, name: &str) -> PyResult> Ok(items) } -/// The effective selection: positional selectors and `--select`, then what every `--select-file` lists, with the files as passed. A file that cannot be read adds no selectors here; the command reports it. -pub(crate) fn effective_selection(args: &Bound<'_, PyAny>) -> PyResult<(Vec, Vec)> { - let mut selectors: Vec = list_attribute(args, "select")?; - let files: Vec = match args.getattr_opt("select_file")? { - Some(value) if !value.is_none() => value - .try_iter()? - .map(|item| escaped_text(&item?.str()?)) - .collect::>()?, - _ => Vec::new(), - }; - for file in &files { - if let Some(text) = file_text(Path::new(file)) { - selectors.extend(selector_file_selectors(&text)); - } - } - Ok((selectors, files)) -} - fn environment(name: &str) -> Option { std::env::var_os(name).map(|value| value.to_string_lossy().into_owned()) } /// `path`'s text when it exists; Recall metadata is best effort, so unreadable means unknown. -fn file_text(path: &Path) -> Option { +pub(crate) fn file_text(path: &Path) -> Option { match std::fs::read_to_string(path) { Ok(text) => Some(text), Err(error) if error.kind() == ErrorKind::NotFound => None, @@ -131,7 +89,7 @@ fn file_text(path: &Path) -> Option { } } -fn project_dir(args: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn project_dir(args: &Bound<'_, PyAny>) -> PyResult { let raw: PathBuf = match text_value(&args.getattr("project_dir")?)? { Some(directory) => PathBuf::from(directory), None => std::env::current_dir()?, @@ -259,6 +217,7 @@ fn invocation_facts( py: Python<'_>, (args, facts): (&Bound<'_, PyAny>, &CommandFacts), (settings, project_dir): (&RecallSettings, &Path), + sqlbuild_version: &str, ) -> PyResult { let fingerprint_source = format!( "{}\u{0}{}\u{0}{}", @@ -289,7 +248,7 @@ fn invocation_facts( destination_database: settings.destination_database.clone(), destination_schema: settings.destination_schema.clone(), destination_fingerprint: Some(sha256_hex(fingerprint_source.as_bytes())), - sqlbuild_version: Some(env!("CARGO_PKG_VERSION").to_owned()), + sqlbuild_version: Some(sqlbuild_version.to_owned()), git_revision: recorded_git_revision(project_dir), git_dirty: recorded_git_dirty(), host_token: host_token(), @@ -334,6 +293,7 @@ fn target_phrase(settings: &RecallSettings) -> String { fn permission_notice( (settings, project_dir): (&RecallSettings, &Path), reason: &str, + sqlbuild_version: &str, ) -> Option { let (dialect, database, place): (Dialect, Option, String) = match &settings.store { StoreLocation::Sqlite { path } => (Dialect::Sqlite, None, format!("at {}", path.display())), @@ -351,7 +311,7 @@ fn permission_notice( database, retention_days: settings.retention_days, cluster: None, - producer_version: env!("CARGO_PKG_VERSION").to_owned(), + producer_version: sqlbuild_version.to_owned(), }; let database_missing: bool = options .database @@ -383,7 +343,7 @@ fn unsupported_notice( } /// Write `text` to stderr; a notice that cannot be written never changes the command's outcome. -fn notify(py: Python<'_>, text: Option) { +pub(crate) fn notify(py: Python<'_>, text: Option) { if let Some(text) = text { drop(write_stderr(py, &text)); } @@ -410,10 +370,11 @@ fn quarantined_notice(path: &Path) -> Option { } } -fn started_session( +pub(crate) fn started_session( py: Python<'_>, (args, identity, facts): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, &CommandFacts), (settings, project_dir, hosted): (&RecallSettings, &Path, Option), + sqlbuild_version: &str, ) -> PyResult>> { let store: SessionStore = match hosted { Some(store) => store, @@ -421,7 +382,10 @@ fn started_session( StoreLocation::Sqlite { path } => { notify(py, quarantined_notice(path)); if let Some(reason) = store_location_problem(path) { - notify(py, permission_notice((settings, project_dir), &reason)); + notify( + py, + permission_notice((settings, project_dir), &reason, sqlbuild_version), + ); return Ok(None); } SessionStore::Sqlite(path.clone()) @@ -429,7 +393,12 @@ fn started_session( StoreLocation::ClickHouse { config, database, .. } => SessionStore::Hosted(Arc::new(ClickHouseStoreOpener { - location: ClickHouseLocation::new(config, database, settings.retention_days), + location: ClickHouseLocation::new( + config, + database, + settings.retention_days, + sqlbuild_version, + ), retention_notice: notice_due, project_dir: project_dir.to_path_buf(), information: Mutex::default(), @@ -449,7 +418,7 @@ fn started_session( let Some(invocation_id) = text_value(&native_field(identity, "invocation_id")?)? else { return Ok(None); }; - let context = invocation_facts(py, (args, facts), (settings, project_dir))?; + let context = invocation_facts(py, (args, facts), (settings, project_dir), sqlbuild_version)?; let session = start_recall_session(SessionConfig { store, maintenance_owner: format!( @@ -460,7 +429,7 @@ fn started_session( envelope: RowEnvelope { invocation_id, invocation_started_at: format_timestamp(SystemTime::now()), - producer_version: env!("CARGO_PKG_VERSION").to_owned(), + producer_version: sqlbuild_version.to_owned(), }, context, retention_days: settings.retention_days, @@ -476,7 +445,7 @@ fn started_session( Ok(Some(Arc::new(session))) } -fn worker_invocation(args: &Bound<'_, PyAny>, facts: &CommandFacts) -> PyResult { +pub(crate) fn worker_invocation(args: &Bound<'_, PyAny>, facts: &CommandFacts) -> PyResult { Ok(facts.command.as_deref() == Some("location") && attribute(args, "location_command")? .extract::>()? @@ -484,7 +453,7 @@ fn worker_invocation(args: &Bound<'_, PyAny>, facts: &CommandFacts) -> PyResult< == Some("execute")) } -fn worker_store(args: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn worker_store(args: &Bound<'_, PyAny>) -> PyResult { let identity = ( attribute(args, "location_run_id")? .extract::()? @@ -507,62 +476,8 @@ fn worker_store(args: &Bound<'_, PyAny>) -> PyResult { }) } -/// Begin recording when Recall is enabled, without waiting on the store. Only the write connection is resolved; a write connection without a read connection is kept for the command to fail with, and other unresolvable settings record nothing. -pub(crate) fn start_invocation_recall( - py: Python<'_>, - args: &Bound<'_, PyAny>, - identity: &Bound<'_, PyAny>, - facts: &CommandFacts, -) -> PyResult { - let project_dir: PathBuf = project_dir(args)?; - let request = SettingsRequest { - project_dir: project_dir.clone(), - selected_target: text_value(&args.getattr("target")?)?, - purpose: StorePurpose::Record, - }; - let mut settings: RecallSettings = match resolve_recall_settings(&request) { - Ok(settings) => settings, - Err(error) if error.invalid_configuration => { - return Ok(InvocationRecall { - configuration_error: Some(error.message), - ..InvocationRecall::default() - }); - } - Err(_unreachable_store) => return Ok(InvocationRecall::default()), - }; - let hosted = if worker_invocation(args, facts)? { - settings.enabled = true; - Some(worker_store(args)?) - } else { - None - }; - let session: Option> = if settings.enabled && settings.project_name.is_some() - { - started_session( - py, - (args, identity, facts), - (&settings, &project_dir, hosted), - )? - } else { - None - }; - let audit_evidence: AuditEvidence = settings.audit_evidence; - Ok(InvocationRecall { - worker: Arc::default(), - session, - target: settings.target.clone(), - adapter: settings.project_adapter.clone(), - command: facts.command.clone().unwrap_or_default(), - settings: Some((settings, project_dir)), - replacements: Arc::default(), - configuration_error: None, - invocation_events: Arc::default(), - audit_evidence, - }) -} - /// `notice` as one Recall line, or block when it already ends with a newline. -fn prefixed(notice: &str) -> String { +pub(crate) fn prefixed(notice: &str) -> String { if notice.ends_with('\n') { format!("Recall: {notice}") } else { @@ -571,7 +486,11 @@ fn prefixed(notice: &str) -> String { } /// What Recall must tell the user about this run once delivery has finished. -fn closing_notices(recall: &InvocationRecall, session: &RecallSession) -> Vec { +pub(crate) fn closing_notices( + recall: &InvocationRecall, + session: &RecallSession, + sqlbuild_version: &str, +) -> Vec { let mut notices: Vec = session .notices() .iter() @@ -580,7 +499,11 @@ fn closing_notices(recall: &InvocationRecall, session: &RecallSession) -> Vec Vec, - recall: &InvocationRecall, - exit_code: Option, -) -> PyResult<()> { - let Some(session) = &recall.session else { - return Ok(()); - }; - session.begin_close(); - while !py.detach(|| session.wait_closed(Duration::from_millis(CLOSE_POLL_MS))) { - if let Err(interrupt) = py.check_signals() { - session.abandon(); - return Err(interrupt); - } - } - let information: Vec = session - .information() - .iter() - .map(|text| prefixed(text)) - .collect(); - let notices: Vec = closing_notices(recall, session); - if !information.is_empty() || !notices.is_empty() { - let affected: Option<&str> = (!notices.is_empty()).then_some(RECALL_HISTORY); - notify( - py, - Some( - information.concat() - + ¬ices.concat() - + &closing_status(&recall.command, exit_code, affected), - ), - ); - } - Ok(()) -} - -/// Forward bytes a process stream accepted to `session` when it records `invocation_id`. -pub(crate) fn forward_output( - session: Option<&RecallSession>, - invocation_id: &str, - (stream, data): (usize, &[u8]), -) { - let Some(session) = session.filter(|session| session.invocation_id() == invocation_id) else { - return; - }; - if in_replacement_sink() { - return; - } - let stream = if stream == STDOUT_STREAM_INDEX { - OutputStream::Stdout - } else { - OutputStream::Stderr - }; - session.append_output(stream, data); -} - -/// Forward one diagnostic's JSON to `session` for `invocation_id`; unparsable ones are skipped. -pub(crate) fn forward_diagnostic( - session: Option<&RecallSession>, - invocation_id: &str, - diagnostic_json: &str, -) { - if let Some(session) = session.filter(|session| session.invocation_id() == invocation_id) { - drop(session.publish_diagnostic(diagnostic_json)); - } -} diff --git a/crates/sqlbuild-python-output/src/recall/_helpers/sinks.rs b/crates/sqlbuild-python-output/src/recall/_helpers/sinks.rs new file mode 100644 index 0000000000..5a4d1caab7 --- /dev/null +++ b/crates/sqlbuild-python-output/src/recall/_helpers/sinks.rs @@ -0,0 +1,41 @@ +//! The native subscriber that publishes every lifecycle fact and artifact into Recall. + +use crate::recall::main::escaped_text::escaped_text; +use std::collections::HashMap; + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use serde_json::{Map, Value}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_recall::records::main::parse_event::parse_event; + +use crate::recall::_helpers::events::event_object; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; + +/// `event` as the JSON object `lifecycle_event_to_json` encodes; `None` when it cannot be encoded. +pub(crate) fn lifecycle_event_object( + py: Python<'_>, + event: &Bound<'_, PyAny>, +) -> PyResult>> { + if let Some(object) = event_object(event)? { + return Ok(Some(object)); + } + let encoded = host_entry(py, ExecutorHost::LifecycleEventToJson)?.call1((event,))?; + let text = escaped_text(encoded.downcast::()?)?; + match parse_event(&text) { + Ok(object) => Ok(Some(object)), + Err(_unencodable) => Ok(None), + } +} + +/// The SQL and resource of a statement that has not reached its terminal event yet. +pub(crate) type RunningSql = HashMap)>; + +/// `outcome` with ordinary exceptions dropped, so recording never changes what the command raises; interrupts still propagate. +pub(crate) fn recording(py: Python<'_>, outcome: PyResult<()>) -> PyResult<()> { + match outcome { + Err(error) if error.is_instance_of::(py) => Ok(()), + other => other, + } +} diff --git a/crates/sqlbuild-python-output/src/recall/_helpers/writer.rs b/crates/sqlbuild-python-output/src/recall/_helpers/writer.rs new file mode 100644 index 0000000000..12d846bf7d --- /dev/null +++ b/crates/sqlbuild-python-output/src/recall/_helpers/writer.rs @@ -0,0 +1,17 @@ +//! `sqlbuild.recall.RecallWriter`: what a Recall replacement sink publishes through. + +use std::sync::Mutex; + +use serde_json::{Map, Value}; + +/// The one stream a writer publishes to. +#[derive(Debug)] +pub(crate) enum Target { + /// The events the sink published for the one it was handed. + Lifecycle(Mutex>>), + /// The chunk the sink was handed, by sequence, and what it published for it. + Output { + sequence: u64, + published: Mutex>>, + }, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python-output/src/recall/constants.rs similarity index 85% rename from crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs rename to crates/sqlbuild-python-output/src/recall/constants.rs index 615b0a4fa3..ebebdd0959 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python-output/src/recall/constants.rs @@ -1,54 +1,78 @@ -//! Recall binding names, environment variables and error codes. +//! Public constants of recall. /// Seconds between heartbeats; tests shorten it to observe abandoned invocations quickly. pub(crate) const HEARTBEAT_ENVIRONMENT: &str = "SQLBUILD_RECALL_HEARTBEAT_SECONDS"; + /// The commit a deployment without `.git` records as `git_revision`. pub(crate) const GIT_REVISION_ENVIRONMENT: &str = "SQLBUILD_GIT_REVISION"; + /// Whether that deployment's tree had uncommitted changes, recorded as `git_dirty`. pub(crate) const GIT_DIRTY_ENVIRONMENT: &str = "SQLBUILD_GIT_DIRTY"; + /// Marker files under `target/` limiting store notices to one per interval. pub(crate) const DDL_NOTICE_PREFIX: &str = ".recall-notice-"; + pub(crate) const NOTICE_INTERVAL_SECONDS: u64 = 24 * 60 * 60; + /// Hex digits of the hostname digest recorded as the host token. pub(crate) const HOST_TOKEN_LENGTH: usize = 16; + /// The shortest heartbeat interval the test environment variable may request. pub(crate) const MINIMUM_HEARTBEAT_SECONDS: f64 = 0.01; + /// What a Recall closing notice says is the only thing affected. pub(crate) const RECALL_HISTORY: &str = "Recall history"; + /// The database the ClickHouse client takes as none, so the server uses the user's default: a store whose database is missing connects with it to create or report the database. pub(crate) const SERVER_DEFAULT_DATABASE: &str = "__default__"; + /// The compute-log stream index of stdout; every other output index is stderr. pub(crate) const STDOUT_STREAM_INDEX: usize = 0; -pub(crate) const RECALL_DISABLED_CODE: &str = "C970"; -pub(crate) const RECALL_STORE_CODE: &str = "C971"; -pub(crate) const RECALL_NOT_FOUND_CODE: &str = "C972"; -pub(crate) const RECALL_AMBIGUOUS_CODE: &str = "C973"; -pub(crate) const RECALL_QUERY_CODE: &str = "C974"; -pub(crate) const RECALL_SETTINGS_CODE: &str = "C975"; -pub(crate) const WAREHOUSE_BACKEND: &str = "warehouse"; + +pub const RECALL_DISABLED_CODE: &str = "C970"; + +pub const RECALL_STORE_CODE: &str = "C971"; + +pub const RECALL_NOT_FOUND_CODE: &str = "C972"; + +pub const RECALL_AMBIGUOUS_CODE: &str = "C973"; + +pub const RECALL_QUERY_CODE: &str = "C974"; + +pub const RECALL_SETTINGS_CODE: &str = "C975"; + +pub const WAREHOUSE_BACKEND: &str = "warehouse"; + /// SQLBuild's state relations in the warehouse; a prefix comparison needs no `LIKE` escape, whose syntax differs by dialect. -pub(crate) const WAREHOUSE_STATE_RELATIONS_SQL: &str = "SELECT table_schema, table_name, table_type \ +pub const WAREHOUSE_STATE_RELATIONS_SQL: &str = "SELECT table_schema, table_name, table_type \ FROM information_schema.tables WHERE SUBSTRING(table_name, 1, 10) = '_sqlbuild_' \ ORDER BY table_schema, table_name"; + /// Default cell width in text formats, in bytes. -pub(crate) const DEFAULT_MAX_CELL_BYTES: usize = 4096; -pub(crate) const JSON_FORMAT: &str = "json"; +pub const DEFAULT_MAX_CELL_BYTES: usize = 4096; + +pub const JSON_FORMAT: &str = "json"; + /// How text output shows SQL NULL. -pub(crate) const NULL_TEXT: &str = "NULL"; +pub const NULL_TEXT: &str = "NULL"; + /// Join keys `sqb recall schema` points out. -pub(crate) const JOIN_KEYS: [&str; 5] = [ +pub const JOIN_KEYS: [&str; 5] = [ "invocation_id", "statement_id", "sql_digest", "run_id", "resource_id", ]; + /// Rows `sqb recall query` returns without `--limit` or `--no-limit`. -pub(crate) const DEFAULT_QUERY_LIMIT: i64 = 100; +pub const DEFAULT_QUERY_LIMIT: i64 = 100; + /// The invocation selector naming the most recently started invocation. -pub(crate) const LAST_INVOCATION: &str = "last"; +pub const LAST_INVOCATION: &str = "last"; + /// The keyword arguments the `sqb recall` dispatch route passes. -pub(crate) const RECALL_OPTIONS: [&str; 14] = [ +pub const RECALL_OPTIONS: [&str; 14] = [ "project_dir", "selected_target", "recall_command", @@ -64,6 +88,7 @@ pub(crate) const RECALL_OPTIONS: [&str; 14] = [ "tail", "ddl", ]; + /// Required text envelope fields of a known lifecycle event. pub(crate) const EVENT_TEXT_FIELDS: [&str; 5] = [ "event_id", @@ -72,6 +97,7 @@ pub(crate) const EVENT_TEXT_FIELDS: [&str; 5] = [ "producer_version", "invocation_id", ]; + /// Optional correlation fields of a known lifecycle event. pub(crate) const EVENT_OPTIONAL_TEXT_FIELDS: [&str; 5] = [ "run_id", @@ -80,8 +106,10 @@ pub(crate) const EVENT_OPTIONAL_TEXT_FIELDS: [&str; 5] = [ "operation_id", "statement_id", ]; + /// The first lifecycle schema version carrying invocation sequence and external context. pub(crate) const INVOCATION_METADATA_SCHEMA_VERSION: i64 = 2; + /// `datetime` attributes of an ISO-8601 timestamp, most significant first. pub(crate) const TIMESTAMP_PARTS: [&str; 7] = [ "year", @@ -92,33 +120,51 @@ pub(crate) const TIMESTAMP_PARTS: [&str; 7] = [ "second", "microsecond", ]; + /// `str(datetime.UTC)`. pub(crate) const UTC_ZONE_NAME: &str = "UTC"; + /// How often closing checks for an interrupt while delivery finishes. pub(crate) const CLOSE_POLL_MS: u64 = 50; + /// Longest wait at startup for another process moving a corrupt store aside; the writer thread retries within the shutdown budget. pub(crate) const QUARANTINE_START_WAIT_MS: u64 = 200; + /// The ClickHouse error name of a query naming a relation that does not exist yet. pub(crate) const UNKNOWN_TABLE_ERROR: &str = "UNKNOWN_TABLE"; + /// `sqb recall query` runs read-only: the setting, and the value that still allows query settings. Not sent when the session is already read-only. pub(crate) const READONLY_SETTING: &str = "readonly"; + pub(crate) const READ_ONLY_QUERIES: i64 = 2; + /// Seconds in a day, for the artifact refresh cutoff. pub(crate) const SECONDS_PER_DAY: u64 = 24 * 60 * 60; + /// The parameter a lifecycle-event sink takes its event in. pub(crate) const EVENT_PARAMETER: &str = "event"; + /// The parameter a command-output sink takes its record in. pub(crate) const RECORD_PARAMETER: &str = "record"; + /// How Recall timestamps mark UTC, and the offset `datetime.fromisoformat` reads. pub(crate) const UTC_SUFFIX: &str = "Z"; + pub(crate) const UTC_OFFSET: &str = "+00:00"; + /// The command whose flags name its Recall `command_mode`. pub(crate) const DEBUG_COMMAND: &str = "debug"; + /// The namespace attribute of `sqb debug --no-connection`. pub(crate) const NO_CONNECTION_ATTRIBUTE: &str = "no_connection"; + /// The statement terminal events, at which the sink settles a statement's SQL. pub(crate) const INVOCATION_STARTED: &str = "invocation_started"; + pub(crate) const INVOCATION_COMPLETED: &str = "invocation_completed"; + pub(crate) const INVOCATION_FAILED: &str = "invocation_failed"; + pub(crate) const STATEMENT_COMPLETED: &str = "statement_completed"; + pub(crate) const STATEMENT_FAILED: &str = "statement_failed"; diff --git a/crates/sqlbuild-python-output/src/recall/main/effective_selection.rs b/crates/sqlbuild-python-output/src/recall/main/effective_selection.rs new file mode 100644 index 0000000000..bae7c78229 --- /dev/null +++ b/crates/sqlbuild-python-output/src/recall/main/effective_selection.rs @@ -0,0 +1,26 @@ +//! The effective selection: positional selectors and `--select`, then what every `--select-file` lists, with the files as passed. A file that cannot be read adds no selectors here; the command reports it. + +use crate::recall::_helpers::session::file_text; +use crate::recall::_helpers::session::list_attribute; +use crate::recall::main::escaped_text::escaped_text; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_cli::dispatch::main::selector_file_selectors::selector_file_selectors; +use std::path::Path; + +/// The effective selection: positional selectors and `--select`, then what every `--select-file` lists, with the files as passed. A file that cannot be read adds no selectors here; the command reports it. +pub fn effective_selection(args: &Bound<'_, PyAny>) -> PyResult<(Vec, Vec)> { + let mut selectors: Vec = list_attribute(args, "select")?; + let files: Vec = match args.getattr_opt("select_file")? { + Some(value) if !value.is_none() => value + .try_iter()? + .map(|item| escaped_text(&item?.str()?)) + .collect::>()?, + _ => Vec::new(), + }; + for file in &files { + if let Some(text) = file_text(Path::new(file)) { + selectors.extend(selector_file_selectors(&text)); + } + } + Ok((selectors, files)) +} diff --git a/crates/sqlbuild-python-output/src/recall/main/escaped_text.rs b/crates/sqlbuild-python-output/src/recall/main/escaped_text.rs new file mode 100644 index 0000000000..d290d2733e --- /dev/null +++ b/crates/sqlbuild-python-output/src/recall/main/escaped_text.rs @@ -0,0 +1,13 @@ +//! The text of `text`; lone surrogates, which UTF-8 cannot hold, are kept as `\\xNN` or `\\uXXXX` escapes. + +use pyo3::prelude::{Bound, PyResult}; +use pyo3::types::{PyString, PyStringMethods}; +use sqlbuild_python_front::discovery::main::snapshots::escaped::escaped; + +/// The text of `text`; lone surrogates, which UTF-8 cannot hold, are kept as `\\xNN` or `\\uXXXX` escapes. +pub fn escaped_text(text: &Bound<'_, PyString>) -> PyResult { + match text.to_str() { + Ok(valid) => Ok(valid.to_owned()), + Err(_) => escaped(text), + } +} diff --git a/crates/sqlbuild-python-output/src/recall/main/finish_invocation_recall.rs b/crates/sqlbuild-python-output/src/recall/main/finish_invocation_recall.rs new file mode 100644 index 0000000000..807836ac9b --- /dev/null +++ b/crates/sqlbuild-python-output/src/recall/main/finish_invocation_recall.rs @@ -0,0 +1,48 @@ +//! Close the session within its shutdown budget; a further interrupt stops waiting at once. Notices end with the command's own outcome. + +use crate::recall::_helpers::session::closing_notices; +use crate::recall::_helpers::session::notify; +use crate::recall::_helpers::session::prefixed; +use crate::recall::constants::CLOSE_POLL_MS; +use crate::recall::constants::RECALL_HISTORY; +use crate::recall::models::InvocationRecall; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_cli::dispatch::main::closing_status::closing_status; +use std::time::Duration; + +/// Close the session within its shutdown budget; a further interrupt stops waiting at once. Notices end with the command's own outcome. +pub fn finish_invocation_recall( + py: Python<'_>, + recall: &InvocationRecall, + exit_code: Option, + sqlbuild_version: &str, +) -> PyResult<()> { + let Some(session) = &recall.session else { + return Ok(()); + }; + session.begin_close(); + while !py.detach(|| session.wait_closed(Duration::from_millis(CLOSE_POLL_MS))) { + if let Err(interrupt) = py.check_signals() { + session.abandon(); + return Err(interrupt); + } + } + let information: Vec = session + .information() + .iter() + .map(|text| prefixed(text)) + .collect(); + let notices: Vec = closing_notices(recall, session, sqlbuild_version); + if !information.is_empty() || !notices.is_empty() { + let affected: Option<&str> = (!notices.is_empty()).then_some(RECALL_HISTORY); + notify( + py, + Some( + information.concat() + + ¬ices.concat() + + &closing_status(&recall.command, exit_code, affected), + ), + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-python-output/src/recall/main/forward_diagnostic.rs b/crates/sqlbuild-python-output/src/recall/main/forward_diagnostic.rs new file mode 100644 index 0000000000..3d81e245c5 --- /dev/null +++ b/crates/sqlbuild-python-output/src/recall/main/forward_diagnostic.rs @@ -0,0 +1,14 @@ +//! Forward one diagnostic's JSON to `session` for `invocation_id`; unparsable ones are skipped. + +use sqlbuild_recall::writer::models::RecallSession; + +/// Forward one diagnostic's JSON to `session` for `invocation_id`; unparsable ones are skipped. +pub fn forward_diagnostic( + session: Option<&RecallSession>, + invocation_id: &str, + diagnostic_json: &str, +) { + if let Some(session) = session.filter(|session| session.invocation_id() == invocation_id) { + drop(session.publish_diagnostic(diagnostic_json)); + } +} diff --git a/crates/sqlbuild-python-output/src/recall/main/forward_output.rs b/crates/sqlbuild-python-output/src/recall/main/forward_output.rs new file mode 100644 index 0000000000..831ee69bc5 --- /dev/null +++ b/crates/sqlbuild-python-output/src/recall/main/forward_output.rs @@ -0,0 +1,26 @@ +//! Forward bytes a process stream accepted to `session` when it records `invocation_id`. + +use crate::recall::_helpers::replacements::in_replacement_sink; +use crate::recall::constants::STDOUT_STREAM_INDEX; +use sqlbuild_recall::records::models::OutputStream; +use sqlbuild_recall::writer::models::RecallSession; + +/// Forward bytes a process stream accepted to `session` when it records `invocation_id`. +pub fn forward_output( + session: Option<&RecallSession>, + invocation_id: &str, + (stream, data): (usize, &[u8]), +) { + let Some(session) = session.filter(|session| session.invocation_id() == invocation_id) else { + return; + }; + if in_replacement_sink() { + return; + } + let stream = if stream == STDOUT_STREAM_INDEX { + OutputStream::Stdout + } else { + OutputStream::Stderr + }; + session.append_output(stream, data); +} diff --git a/crates/sqlbuild-python-output/src/recall/main/mod.rs b/crates/sqlbuild-python-output/src/recall/main/mod.rs new file mode 100644 index 0000000000..39abeaa3ae --- /dev/null +++ b/crates/sqlbuild-python-output/src/recall/main/mod.rs @@ -0,0 +1,9 @@ +pub mod effective_selection; +pub mod escaped_text; +pub mod finish_invocation_recall; +pub mod forward_diagnostic; +pub mod forward_output; +pub mod publish_lifecycle_event; +pub mod start_invocation_recall; +pub mod write_stderr; +pub mod write_stdout_bytes; diff --git a/crates/sqlbuild-python-output/src/recall/main/publish_lifecycle_event.rs b/crates/sqlbuild-python-output/src/recall/main/publish_lifecycle_event.rs new file mode 100644 index 0000000000..5891a47c2a --- /dev/null +++ b/crates/sqlbuild-python-output/src/recall/main/publish_lifecycle_event.rs @@ -0,0 +1,17 @@ +//! Queue `event` in `session`; an event that cannot be encoded is skipped. + +use crate::recall::_helpers::sinks::lifecycle_event_object; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_recall::writer::models::RecallSession; + +/// Queue `event` in `session`; an event that cannot be encoded is skipped. +pub fn publish_lifecycle_event( + py: Python<'_>, + session: &RecallSession, + event: &Bound<'_, PyAny>, +) -> PyResult<()> { + if let Some(object) = lifecycle_event_object(py, event)? { + session.publish_event_object(object); + } + Ok(()) +} diff --git a/crates/sqlbuild-python-output/src/recall/main/start_invocation_recall.rs b/crates/sqlbuild-python-output/src/recall/main/start_invocation_recall.rs new file mode 100644 index 0000000000..2d890aae0b --- /dev/null +++ b/crates/sqlbuild-python-output/src/recall/main/start_invocation_recall.rs @@ -0,0 +1,71 @@ +//! Begin recording when Recall is enabled, without waiting on the store. Only the write connection is resolved; a write connection without a read connection is kept for the command to fail with, and other unresolvable settings record nothing. + +use crate::recall::_helpers::session::project_dir; +use crate::recall::_helpers::session::started_session; +use crate::recall::_helpers::session::text_value; +use crate::recall::_helpers::session::worker_invocation; +use crate::recall::_helpers::session::worker_store; +use crate::recall::models::InvocationRecall; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_cli::dispatch::models::CommandFacts; +use sqlbuild_recall::settings::main::resolve_recall_settings::resolve_recall_settings; +use sqlbuild_recall::settings::models::{ + AuditEvidence, RecallSettings, SettingsRequest, StorePurpose, +}; +use sqlbuild_recall::writer::models::RecallSession; +use std::path::PathBuf; +use std::sync::Arc; + +/// Begin recording when Recall is enabled, without waiting on the store. Only the write connection is resolved; a write connection without a read connection is kept for the command to fail with, and other unresolvable settings record nothing. +pub fn start_invocation_recall( + py: Python<'_>, + (args, identity, facts): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, &CommandFacts), + sqlbuild_version: &str, +) -> PyResult { + let project_dir: PathBuf = project_dir(args)?; + let request = SettingsRequest { + project_dir: project_dir.clone(), + selected_target: text_value(&args.getattr("target")?)?, + purpose: StorePurpose::Record, + }; + let mut settings: RecallSettings = match resolve_recall_settings(&request) { + Ok(settings) => settings, + Err(error) if error.invalid_configuration => { + return Ok(InvocationRecall { + configuration_error: Some(error.message), + ..InvocationRecall::default() + }); + } + Err(_unreachable_store) => return Ok(InvocationRecall::default()), + }; + let hosted = if worker_invocation(args, facts)? { + settings.enabled = true; + Some(worker_store(args)?) + } else { + None + }; + let session: Option> = if settings.enabled && settings.project_name.is_some() + { + started_session( + py, + (args, identity, facts), + (&settings, &project_dir, hosted), + sqlbuild_version, + )? + } else { + None + }; + let audit_evidence: AuditEvidence = settings.audit_evidence; + Ok(InvocationRecall { + worker: Arc::default(), + session, + target: settings.target.clone(), + adapter: settings.project_adapter.clone(), + command: facts.command.clone().unwrap_or_default(), + settings: Some((settings, project_dir)), + replacements: Arc::default(), + configuration_error: None, + invocation_events: Arc::default(), + audit_evidence, + }) +} diff --git a/crates/sqlbuild-python-output/src/recall/main/write_stderr.rs b/crates/sqlbuild-python-output/src/recall/main/write_stderr.rs new file mode 100644 index 0000000000..bdffd3aaac --- /dev/null +++ b/crates/sqlbuild-python-output/src/recall/main/write_stderr.rs @@ -0,0 +1,9 @@ +//! Write `text` to stderr and flush it. + +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_python_base::native_host::main::write_console::write_console; + +/// Write `text` to stderr and flush it. +pub fn write_stderr(py: Python<'_>, text: &str) -> PyResult<()> { + write_console(py, "stderr", text) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs b/crates/sqlbuild-python-output/src/recall/main/write_stdout_bytes.rs similarity index 55% rename from crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs rename to crates/sqlbuild-python-output/src/recall/main/write_stdout_bytes.rs index 54e837c297..0b8595d68a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs +++ b/crates/sqlbuild-python-output/src/recall/main/write_stdout_bytes.rs @@ -1,12 +1,10 @@ -//! Recall's writes to the process streams, through Python so captures and tees see them. +//! Write `data` unchanged to stdout's binary buffer after flushing pending text. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyBytes; -use sqlbuild_python_base::native_host::main::write_console::write_console; - /// Write `data` unchanged to stdout's binary buffer after flushing pending text. -pub(crate) fn write_stdout_bytes(py: Python<'_>, data: &[u8]) -> PyResult<()> { +pub fn write_stdout_bytes(py: Python<'_>, data: &[u8]) -> PyResult<()> { let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; stdout.call_method0("flush")?; let target: Bound<'_, PyAny> = stdout.getattr("buffer").unwrap_or(stdout); @@ -14,8 +12,3 @@ pub(crate) fn write_stdout_bytes(py: Python<'_>, data: &[u8]) -> PyResult<()> { target.call_method0("flush")?; Ok(()) } - -/// Write `text` to stderr and flush it. -pub(crate) fn write_stderr(py: Python<'_>, text: &str) -> PyResult<()> { - write_console(py, "stderr", text) -} diff --git a/crates/sqlbuild-python-output/src/recall/mod.rs b/crates/sqlbuild-python-output/src/recall/mod.rs new file mode 100644 index 0000000000..f40852fd4c --- /dev/null +++ b/crates/sqlbuild-python-output/src/recall/mod.rs @@ -0,0 +1,6 @@ +//! The Recall session: its writer, sinks, events, console output, replacements and the ClickHouse store. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-output/src/recall/models.rs b/crates/sqlbuild-python-output/src/recall/models.rs new file mode 100644 index 0000000000..8fae21245a --- /dev/null +++ b/crates/sqlbuild-python-output/src/recall/models.rs @@ -0,0 +1,781 @@ +//! Public records of recall. + +use crate::recall::_helpers::clickhouse_calls as calls; +use crate::recall::_helpers::clickhouse_calls::ConnectionRole; +use crate::recall::_helpers::clickhouse_store::ClickHouseStore; +use crate::recall::_helpers::clickhouse_store::StoreState; +use crate::recall::_helpers::clickhouse_store::connect_store; +use crate::recall::_helpers::clickhouse_store::integer; +use crate::recall::_helpers::clickhouse_store::literal; +use crate::recall::_helpers::clickhouse_store::observed_version; +use crate::recall::_helpers::clickhouse_store::text; +use crate::recall::_helpers::replacements::LifecycleReplacement; +use crate::recall::_helpers::replacements::OutputSink; +use crate::recall::_helpers::replacements::PendingEvent; +use crate::recall::_helpers::replacements::discovered; +use crate::recall::_helpers::sinks::RunningSql; +use crate::recall::_helpers::sinks::lifecycle_event_object; +use crate::recall::_helpers::sinks::recording; +use crate::recall::_helpers::writer::Target; +use crate::recall::constants::INVOCATION_COMPLETED; +use crate::recall::constants::INVOCATION_FAILED; +use crate::recall::constants::INVOCATION_STARTED; +use crate::recall::constants::READ_ONLY_QUERIES; +use crate::recall::constants::READONLY_SETTING; +use crate::recall::constants::STATEMENT_COMPLETED; +use crate::recall::constants::STATEMENT_FAILED; +use crate::recall::constants::UNKNOWN_TABLE_ERROR; +use crate::recall::main::escaped_text::escaped_text; +use crate::recall::main::publish_lifecycle_event::publish_lifecycle_event; +use pyo3::exceptions::PyTypeError; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::Bound; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::prelude::pyclass; +use pyo3::prelude::pymethods; +use pyo3::types::PyBytes; +use pyo3::types::PyBytesMethods; +use pyo3::types::PyDict; +use pyo3::types::PyString; +use serde_json::Map; +use serde_json::Value; +use sqlbuild_config::models::ConfigValue; +use sqlbuild_core::event_export::main::route_lifecycle_event::route_lifecycle_event; +use sqlbuild_core::event_export::models::ExporterCounts; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::event_exporting::main::export_policy::export_policy; +use sqlbuild_python_base::event_exporting::models::BoundEventExporter; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::project_snapshot::models::WorkerBuild; +use sqlbuild_recall::records::models::OutputStream; +use sqlbuild_recall::records::models::RecallValue; +use sqlbuild_recall::schema::constants::ALL_TABLES; +use sqlbuild_recall::schema::constants::VIEW_NAMES; +use sqlbuild_recall::schema::main::clickhouse_relation::clickhouse_relation; +use sqlbuild_recall::schema::main::create_database_statement::create_database_statement; +use sqlbuild_recall::schema::main::ddl_statements::ddl_statements; +use sqlbuild_recall::schema::main::table_spec::table_spec; +use sqlbuild_recall::schema::main::upgrade_statements::upgrade_statements; +use sqlbuild_recall::schema::models::DdlOptions; +use sqlbuild_recall::schema::models::Dialect; +use sqlbuild_recall::schema::models::Table; +use sqlbuild_recall::settings::models::AuditEvidence; +use sqlbuild_recall::settings::models::RecallSettings; +use sqlbuild_recall::store::constants::LAST_INVOCATION_SELECTOR; +use sqlbuild_recall::store::constants::LISTED_INVOCATION_CANDIDATES; +use sqlbuild_recall::store::errors::StoreError; +use sqlbuild_recall::store::errors::StoreErrorKind; +use sqlbuild_recall::store::main::chosen_invocation::chosen_invocation; +use sqlbuild_recall::store::models::DescribedColumn; +use sqlbuild_recall::store::models::DescribedRelation; +use sqlbuild_recall::store::models::OutputTranscript; +use sqlbuild_recall::store::models::QueryResult; +use sqlbuild_recall::store::types::RecallStore; +use sqlbuild_recall::writer::models::DeliveryStream; +use sqlbuild_recall::writer::models::RecallSession; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::OnceLock; +use std::sync::PoisonError; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; + +/// What `sqb recall query` reports about a ClickHouse query besides its rows. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct QueryStatistics { + pub rows_read: Option, + pub bytes_read: Option, + pub query_id: Option, +} + +/// Where a ClickHouse store lives and how long it keeps history. +#[derive(Clone, Debug)] +pub struct ClickHouseLocation { + pub config: Vec<(String, ConfigValue)>, + pub options: DdlOptions, +} + +impl ClickHouseLocation { + pub fn new( + config: &[(String, ConfigValue)], + database: &str, + retention: u32, + producer_version: &str, + ) -> Self { + Self { + config: config.to_vec(), + options: DdlOptions { + database: Some(database.to_owned()), + retention_days: retention, + cluster: None, + producer_version: producer_version.to_owned(), + }, + } + } + + pub(crate) fn connect_action(&self) -> String { + format!("connect to the Recall store in `{}`", self.database()) + } + + pub(crate) fn database(&self) -> &str { + self.options.database.as_deref().unwrap_or_default() + } + + pub(crate) fn table(&self, name: &str) -> String { + clickhouse_relation(name, &self.options) + } +} + +/// A ClickHouse store opened for `sqb recall`. +#[derive(Debug)] +pub struct ClickHouseReader { + pub(crate) store: ClickHouseStore, +} + +impl ClickHouseReader { + pub fn open(py: Python<'_>, location: &ClickHouseLocation) -> Result { + let store = ClickHouseStore { + location: location.clone(), + connection: Some(calls::connect( + py, + (&location.config, ConnectionRole::Reader), + &location.connect_action(), + )?), + columns: HashMap::new(), + notices: Vec::new(), + information: Vec::new(), + }; + Ok(Self { store }) + } + + /// Fail with "no history yet" when the store's tables or views are missing; queries check only after they fail, so a successful one costs no extra round trip. + pub fn require_store(&self, py: Python<'_>) -> Result<(), StoreError> { + let state: StoreState = self.store.state(py)?; + if state.tables.len() < ALL_TABLES.len() || state.views.len() < VIEW_NAMES.len() { + return Err(StoreError { + kind: StoreErrorKind::NotFound, + message: self.not_created_message(), + }); + } + Ok(()) + } + + pub(crate) fn not_created_message(&self) -> String { + format!( + "no Recall history is recorded yet in ClickHouse database `{}`, or its tables are \ + still being created; run a command first", + self.database() + ) + } + + /// The statements that create the store in `location`, or bring it up to this build's schema, read with `SELECT` only. + pub fn upgrade_statements( + py: Python<'_>, + location: &ClickHouseLocation, + ) -> Result, StoreError> { + let mut store = ClickHouseStore { + location: location.clone(), + connection: Some(connect_store(py, location, ConnectionRole::Reader)?), + columns: HashMap::new(), + notices: Vec::new(), + information: Vec::new(), + }; + match store.database_exists(py) { + Ok(true) => {} + Ok(false) => { + store.close(); + return Ok(create_database_statement(&location.options) + .into_iter() + .chain(ddl_statements(Dialect::ClickHouse, &location.options)) + .collect()); + } + Err(error) => { + store.close(); + return Err(error); + } + } + let observed = { + let state: StoreState = store.state(py)?; + store.read_columns(py)?; + let version: i64 = if state.tables.len() < ALL_TABLES.len() { + state.version + } else { + observed_version(store.store_version(py, &state)?) + }; + Ok(store.observed(&state.views, version)) + }; + store.close(); + Ok(upgrade_statements( + Dialect::ClickHouse, + &location.options, + &observed?, + )) + } + + /// The database the store lives in. + pub fn database(&self) -> &str { + self.store.location.database() + } + + /// At most `limit` rows of a read-only query, with the server's statistics. A failure on a store that is not created yet is a store failure (C971), not an invalid query. An unknown relation on a complete store is retried once, since its views may have been created since the query ran; a relation still unknown is the query's own error (C974). + pub fn query( + &self, + py: Python<'_>, + sql: &str, + limit: Option, + ) -> Result<(QueryResult, QueryStatistics), StoreError> { + let settings: [(&str, i64); 1] = [(READONLY_SETTING, READ_ONLY_QUERIES)]; + let row_cap: Option = limit.map(|limit| limit.saturating_add(1)); + let run = || { + calls::query( + py, + self.store.connection()?, + (sql, &settings, row_cap), + "run the Recall query", + ) + }; + let rejected = |failure: StoreError| StoreError { + kind: if failure.kind == StoreErrorKind::Busy { + StoreErrorKind::Other + } else { + StoreErrorKind::Query + }, + message: failure.message, + }; + let (mut result, statistics) = match run() { + Ok(found) => found, + Err(failure) => match self.require_store(py) { + Err(missing) if missing.kind == StoreErrorKind::NotFound => { + return Err(StoreError { + kind: StoreErrorKind::Other, + message: missing.message, + }); + } + Ok(()) if failure.message.contains(UNKNOWN_TABLE_ERROR) => { + run().map_err(rejected)? + } + Ok(()) | Err(_) => return Err(rejected(failure)), + }, + }; + if let Some(limit) = limit + && result.rows.len() > limit + { + result.rows.truncate(limit); + result.truncated = true; + } + Ok((result, statistics)) + } + + /// The store's views, then its tables, with their columns. + pub fn describe(&self, py: Python<'_>) -> Result, StoreError> { + let names: Vec = VIEW_NAMES + .iter() + .map(|name| literal(name)) + .chain( + ALL_TABLES + .iter() + .map(|table| literal(table_spec(*table).name)), + ) + .collect(); + let found = self.store.query( + py, + &format!( + "SELECT c.table, if(t.engine = 'View', 'view', 'table') AS kind, c.name, c.type \ + FROM system.columns AS c INNER JOIN system.tables AS t \ + ON t.database = c.database AND t.name = c.table \ + WHERE c.database = {} AND c.table IN ({}) \ + ORDER BY kind = 'table', c.table, c.position", + literal(self.database()), + names.join(", ") + ), + "describe the Recall store", + )?; + let mut relations: Vec = Vec::new(); + for row in &found.rows { + let name = text(row.first()); + if relations.last().is_none_or(|last| last.name != name) { + relations.push(DescribedRelation { + name, + kind: text(row.get(1)), + columns: Vec::new(), + }); + } + let declared_type = text(row.get(3)); + if let Some(relation) = relations.last_mut() { + relation.columns.push(DescribedColumn { + name: text(row.get(2)), + nullable: declared_type.starts_with("Nullable("), + declared_type, + }); + } + } + Ok(relations) + } + + /// The full invocation ID an ID, unique prefix or `last` names. + pub fn resolve_invocation(&self, py: Python<'_>, selector: &str) -> Result { + let location = &self.store.location; + let started = format!( + "(SELECT invocation_id, invocation_started_at FROM {} \ + UNION ALL SELECT invocation_id, invocation_started_at FROM {})", + location.table(table_spec(Table::Invocations).name), + location.table(table_spec(Table::Heartbeats).name), + ); + let sql = if selector == LAST_INVOCATION_SELECTOR { + format!( + "SELECT invocation_id FROM {started} \ + ORDER BY invocation_started_at DESC, invocation_id DESC LIMIT 1" + ) + } else { + format!( + "SELECT DISTINCT invocation_id FROM {started} \ + WHERE startsWith(invocation_id, {}) ORDER BY invocation_id LIMIT {}", + literal(selector), + LISTED_INVOCATION_CANDIDATES + 1 + ) + }; + let found = self.store.query(py, &sql, "find the invocation")?; + let candidates: Vec = found.rows.iter().map(|row| text(row.first())).collect(); + chosen_invocation(selector, &candidates) + } + + /// One invocation's exact output bytes, deduplicated by sequence, and its completeness. + pub fn read_output_transcript( + &self, + py: Python<'_>, + invocation_id: &str, + stream: Option, + ) -> Result { + let id = literal(invocation_id); + let stream_filter: String = stream.map_or_else(String::new, |stream| { + format!(" AND stream = {}", literal(stream.as_str())) + }); + let output = self.store.location.table(table_spec(Table::Output).name); + let chunks = self.store.query( + py, + &format!( + "SELECT data FROM (SELECT sequence, stream, data FROM {output} \ + WHERE invocation_id = {id} ORDER BY sequence, ingested_at LIMIT 1 BY sequence) \ + WHERE 1{stream_filter} ORDER BY sequence" + ), + "read Recall output", + )?; + let mut transcript = OutputTranscript { + invocation_id: invocation_id.to_owned(), + ..OutputTranscript::default() + }; + for row in &chunks.rows { + match row.first() { + Some(RecallValue::Blob(data)) => transcript.data.extend_from_slice(data), + Some(RecallValue::Text(text)) => transcript.data.extend_from_slice(text.as_bytes()), + _ => {} + } + } + let database = self.store.location.table("invocations"); + let status = self.store.query( + py, + &format!("SELECT status FROM {database} WHERE invocation_id = {id}"), + "read the Recall invocation", + )?; + transcript.status = status.rows.first().map(|row| text(row.first())); + let completeness_view = self.store.location.table("output_invocations"); + let completeness = self.store.query( + py, + &format!( + "SELECT missing_sequences, reported_drops, filtered, flush_complete, complete, \ + gap_reason FROM {completeness_view} WHERE invocation_id = {id}" + ), + "read Recall output completeness", + )?; + if let Some(row) = completeness.rows.first() { + transcript.missing_sequences = integer(row.first()); + transcript.reported_drops = integer(row.get(1)); + transcript.filtered = integer(row.get(2)); + transcript.flush_complete = integer(row.get(3)) != 0; + transcript.complete = integer(row.get(4)) != 0; + let reason = text(row.get(5)); + transcript.gap_reason = (!reason.is_empty()).then_some(reason); + } + Ok(transcript) + } + + pub fn close(mut self) { + self.store.close(); + } +} + +/// The replacement sinks of one invocation, configured once discovery has run. +#[derive(Default)] +pub struct RecallReplacements { + pub(crate) lifecycle: OnceLock>, +} + +impl std::fmt::Debug for RecallReplacements { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RecallReplacements") + .field("lifecycle", &self.lifecycle.get().is_some()) + .finish() + } +} + +impl RecallReplacements { + /// Install the discovered replacements, each `(sink, writer_parameter)` or `None`; whether any filters Recall history. + pub fn configure( + &self, + session: &Arc, + (lifecycle, output): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + ) -> PyResult { + let mut active: bool = false; + if let Some((sink, writer_parameter)) = discovered(lifecycle)? { + let bound = BoundEventExporter::create( + native_field(&sink, "name")?.extract()?, + ( + native_field(&sink, "function")?.unbind(), + PyDict::new(sink.py()).into_any().unbind(), + ), + ( + Some(&native_field(&sink, "event_kinds")?), + Some(&native_field(&sink, "min_severity")?), + ), + )?; + drop(self.lifecycle.set(Arc::new(LifecycleReplacement { + function: bound.function, + writer_parameter, + filter: bound.filter, + }))); + active = true; + } + if let Some((sink, writer_parameter)) = discovered(output)? { + let mut streams: [bool; 2] = [false; 2]; + for stream in native_field(&sink, "streams")?.try_iter()? { + let name: String = native_field(&stream?, "value")?.extract()?; + if let Some(stream) = OutputStream::from_name(&name) { + streams[usize::from(stream == OutputStream::Stderr)] = true; + } + } + session.set_output_replacement(Arc::new(OutputSink { + function: native_field(&sink, "function")?.unbind(), + writer_parameter, + streams, + invocation_id: session.invocation_id(), + })); + active = true; + } + Ok(active) + } + + /// Queue `event` for the lifecycle replacement, which runs on the delivery thread; `false` when there is none. + pub fn deliver_lifecycle( + &self, + py: Python<'_>, + session: &Arc, + event: &Bound<'_, PyAny>, + ) -> PyResult { + let Some(replacement) = self.lifecycle.get() else { + return Ok(false); + }; + let event_type: String = native_field(event, "event_type")?.extract()?; + let eligible: bool = export_policy(py, &event_type).is_ok_and(|policy| { + let (_, eligible) = route_lifecycle_event( + &policy, + std::slice::from_ref(&replacement.filter), + vec![ExporterCounts::default()], + ); + !eligible.is_empty() + }); + if !eligible { + session.record_filtered(DeliveryStream::Events, 1); + return Ok(true); + } + let schema_version: i64 = native_field(event, "schema_version")?.extract()?; + session.publish_pending_event( + Box::new(PendingEvent { + replacement: Arc::clone(replacement), + event: event.clone().unbind(), + }), + (&event_type, schema_version), + ); + Ok(true) + } +} + +/// What the invocation records: its session, when enabled, and the resolved target and adapter. +#[derive(Clone, Debug, Default)] +pub struct InvocationRecall { + /// The verified worker remains owned until Recall has drained and its claim can be released. + pub worker: Arc>>>, + pub session: Option>, + pub target: Option, + pub adapter: Option, + /// The command name, for the closing status line. + pub command: String, + /// The resolved settings and project directory, for notices at the end of the run. + pub settings: Option<(RecallSettings, PathBuf)>, + /// The Recall replacement sinks, configured once runtime extensions are discovered. + pub replacements: Arc, + /// A Recall configuration mistake the command fails with before it runs. + pub configuration_error: Option, + /// The invocation's own lifecycle events the session has recorded. + pub invocation_events: Arc, + /// Which failing audits collect evidence rows, as `recall.audit_evidence` sets it. + pub audit_evidence: AuditEvidence, +} + +/// Which of an invocation's own lifecycle events the sink has recorded, so an interrupted run records each exactly once. +#[derive(Debug, Default)] +pub struct InvocationEvents { + pub(crate) started: AtomicBool, + pub(crate) ended: AtomicBool, +} + +impl InvocationEvents { + /// Whether `invocation_started` was recorded. + pub fn started(&self) -> bool { + self.started.load(Ordering::SeqCst) + } + + /// Whether `invocation_completed` or `invocation_failed` was recorded. + pub fn ended(&self) -> bool { + self.ended.load(Ordering::SeqCst) + } + + pub(crate) fn note(&self, event_type: &str) { + if event_type == INVOCATION_STARTED { + self.started.store(true, Ordering::SeqCst); + } else if event_type == INVOCATION_COMPLETED || event_type == INVOCATION_FAILED { + self.ended.store(true, Ordering::SeqCst); + } + } +} + +/// Publishes lifecycle events, through the Recall replacement sink on the delivery thread when one is configured, and SQL artifacts of the active session; delivery is best effort. The SQL of a statement that fails or is cancelled is queued again with its terminal event's priority. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct RecallLifecycleSink { + pub session: Arc, + pub replacements: Arc, + pub running_sql: Arc>, + pub invocation_events: Arc, +} + +#[pymethods] +impl RecallLifecycleSink { + /// Record `event`; an event Recall cannot encode is skipped, never raised into the command. + pub(crate) fn __call__(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { + recording(py, self.publish(py, event)) + } + + /// Record one SQL artifact; text Recall cannot encode is skipped, never raised into the command. + #[pyo3(signature = (*, sql, statement_id=None, resource_id=None))] + pub(crate) fn record_sql( + &self, + py: Python<'_>, + sql: &Bound<'_, PyString>, + statement_id: Option<&Bound<'_, PyString>>, + resource_id: Option<&Bound<'_, PyString>>, + ) -> PyResult<()> { + let texts = (|| -> PyResult<(String, Option, Option)> { + Ok(( + escaped_text(sql)?, + statement_id.map(escaped_text).transpose()?, + resource_id.map(escaped_text).transpose()?, + )) + })(); + recording( + py, + texts.map(|(sql, statement_id, resource_id)| { + self.session + .publish_sql(&sql, statement_id.as_deref(), resource_id.as_deref()); + if let Some(statement_id) = statement_id { + self.running().insert(statement_id, (sql, resource_id)); + } + }), + ) + } +} + +impl RecallLifecycleSink { + pub(crate) fn running(&self) -> std::sync::MutexGuard<'_, RunningSql> { + self.running_sql + .lock() + .unwrap_or_else(PoisonError::into_inner) + } + + /// Forget a statement's SQL at its terminal event; a failed one's SQL is queued again first. + pub(crate) fn settle_statement( + &self, + event: &Bound<'_, PyAny>, + event_type: &str, + ) -> PyResult<()> { + if event_type != STATEMENT_COMPLETED && event_type != STATEMENT_FAILED { + return Ok(()); + } + let Some(statement_id) = event.getattr_opt("statement_id")? else { + return Ok(()); + }; + if statement_id.is_none() { + return Ok(()); + } + let statement_id: String = escaped_text(&statement_id.str()?)?; + let Some((sql, resource_id)) = self.running().remove(&statement_id) else { + return Ok(()); + }; + if event_type == STATEMENT_FAILED { + self.session + .publish_failed_sql(&sql, Some(&statement_id), resource_id.as_deref()); + } + Ok(()) + } + + pub(crate) fn publish(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { + let event_type: String = match event.getattr_opt("event_type")? { + Some(event_type) => event_type.str()?.to_string(), + None => String::new(), + }; + self.settle_statement(event, &event_type)?; + if !self + .replacements + .deliver_lifecycle(py, &self.session, event)? + { + publish_lifecycle_event(py, &self.session, event)?; + } + self.invocation_events.note(&event_type); + Ok(()) + } +} + +/// Publishes the records a Recall replacement sink keeps; valid only while the sink call runs. +#[pyclass(module = "sqlbuild._native", name = "RecallWriter", frozen)] +#[derive(Debug)] +pub struct RecallWriter { + pub(crate) target: Target, + pub(crate) open: AtomicBool, +} + +impl RecallWriter { + /// A writer for one lifecycle event handed to the lifecycle replacement. + pub fn lifecycle() -> Self { + Self::new(Target::Lifecycle(Mutex::new(Vec::new()))) + } + + /// A writer for the output chunk with `sequence` handed to the output replacement. + pub fn output(sequence: u64) -> Self { + Self::new(Target::Output { + sequence, + published: Mutex::new(None), + }) + } + + pub(crate) fn new(target: Target) -> Self { + Self { + target, + open: AtomicBool::new(true), + } + } + + /// End the sink call: later publishing raises, and the events or output bytes published are returned. + pub fn finish(&self) -> (Vec>, Option>) { + self.open.store(false, Ordering::SeqCst); + match &self.target { + Target::Lifecycle(published) => ( + std::mem::take(&mut *published.lock().unwrap_or_else(PoisonError::into_inner)), + None, + ), + Target::Output { published, .. } => ( + Vec::new(), + published + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(), + ), + } + } + + pub(crate) fn ensure_open(&self) -> PyResult<()> { + if self.open.load(Ordering::SeqCst) { + return Ok(()); + } + Err(PyValueError::new_err( + "a RecallWriter can publish only while the sink call it was passed to runs", + )) + } +} + +#[pymethods] +impl RecallWriter { + /// Record `event` in Recall; for the lifecycle replacement sink. + pub(crate) fn publish_lifecycle( + &self, + py: Python<'_>, + event: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.ensure_open()?; + let Target::Lifecycle(published) = &self.target else { + return Err(PyValueError::new_err( + "this RecallWriter belongs to a command-output sink; use publish_command_output", + )); + }; + if !event.is_instance(&host_entry(py, ExecutorHost::LifecycleEvent)?)? { + return Err(PyTypeError::new_err( + "publish_lifecycle expects a LifecycleEvent", + )); + } + if let Some(object) = lifecycle_event_object(py, event)? { + published + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(object); + } + Ok(()) + } + + /// Record `record.data` in Recall as the output chunk the sink was passed; at most once per call. + pub(crate) fn publish_command_output( + &self, + py: Python<'_>, + record: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.ensure_open()?; + let Target::Output { + sequence, + published, + } = &self.target + else { + return Err(PyValueError::new_err( + "this RecallWriter belongs to a lifecycle-event sink; use publish_lifecycle", + )); + }; + if !record.is_instance(&host_entry(py, ExecutorHost::CommandOutputRecord)?)? { + return Err(PyTypeError::new_err( + "publish_command_output expects a CommandOutputRecord", + )); + } + let record_sequence: u64 = native_field(record, "sequence")?.extract()?; + if record_sequence != *sequence { + return Err(PyValueError::new_err(format!( + "publish_command_output expects the record passed to this call (sequence \ + {sequence}) or a copy made with record.replace(), not sequence {record_sequence}" + ))); + } + let data: Vec = native_field(record, "data")? + .downcast::() + .map_err(|_| PyTypeError::new_err("record.data must be bytes"))? + .as_bytes() + .to_vec(); + let mut slot = published.lock().unwrap_or_else(PoisonError::into_inner); + if slot.is_some() { + return Err(PyValueError::new_err( + "a command-output record can be published once per sink call", + )); + } + *slot = Some(data); + Ok(()) + } +} diff --git a/crates/sqlbuild-python-output/src/result_output/_helpers/events/event_writer.rs b/crates/sqlbuild-python-output/src/result_output/_helpers/events/event_writer.rs new file mode 100644 index 0000000000..c63a5263b1 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/_helpers/events/event_writer.rs @@ -0,0 +1,23 @@ +//! Append canonical integration-result records for a parent integration, one per claimed terminal. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +pub(crate) const BUILD_OPTIONS: (&[&str], &[&str]) = (&["result", "plan"], &["command"]); + +/// A Python context manager entered for the duration of `body`. +pub(crate) fn within( + manager: &Bound<'_, PyAny>, + body: impl FnOnce() -> PyResult, +) -> PyResult { + manager.call_method0("__enter__")?; + let outcome: PyResult = body(); + let none = manager.py().None(); + manager.call_method1("__exit__", (&none, &none, &none))?; + outcome +} + +pub(crate) fn write(stream: &Bound<'_, PyAny>, line: &str) -> PyResult<()> { + stream.call_method1("write", (line,))?; + stream.call_method0("flush")?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs b/crates/sqlbuild-python-output/src/result_output/_helpers/events/integration_envelope.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs rename to crates/sqlbuild-python-output/src/result_output/_helpers/events/integration_envelope.rs index 3689487c6e..7e2640a5e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs +++ b/crates/sqlbuild-python-output/src/result_output/_helpers/events/integration_envelope.rs @@ -1,7 +1,7 @@ //! Decoded integration-result records, read by the integrations that consume them. use pyo3::exceptions::PyAttributeError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; use pyo3::{pyclass, pymethods}; use sqlbuild_core::integration_results::errors::IntegrationRecordError; @@ -187,9 +187,3 @@ impl IntegrationResultEnvelope { field(&self.fields, py, name) } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs b/crates/sqlbuild-python-output/src/result_output/_helpers/events/integration_records.rs similarity index 97% rename from crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs rename to crates/sqlbuild-python-output/src/result_output/_helpers/events/integration_records.rs index 8c098250dc..560e14b717 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs +++ b/crates/sqlbuild-python-output/src/result_output/_helpers/events/integration_records.rs @@ -14,13 +14,13 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::result_output::assets::microbatch_summary; -use crate::bindings::_helpers::result_output::checks::{ - measured_value, measurement_thresholds, python_check_status, -}; -use crate::bindings::_helpers::result_output::projection::{ - ResultIdentity, ResultKind, result_kind, -}; +use crate::result_output::_helpers::results::assets::microbatch_summary; +use crate::result_output::_helpers::results::checks::measured_value; +use crate::result_output::_helpers::results::checks::measurement_thresholds; +use crate::result_output::_helpers::results::checks::python_check_status; +use crate::result_output::_helpers::results::projection::ResultIdentity; +use crate::result_output::_helpers::results::projection::ResultKind; +use crate::result_output::_helpers::results::projection::result_kind; use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; diff --git a/crates/sqlbuild-python-output/src/result_output/_helpers/events/mod.rs b/crates/sqlbuild-python-output/src/result_output/_helpers/events/mod.rs new file mode 100644 index 0000000000..727c48aa30 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/_helpers/events/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod event_writer; +pub(crate) mod integration_envelope; +pub(crate) mod integration_records; +pub(crate) mod resource_ids; +pub(crate) mod terminal_index; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs b/crates/sqlbuild-python-output/src/result_output/_helpers/events/resource_ids.rs similarity index 57% rename from crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs rename to crates/sqlbuild-python-output/src/result_output/_helpers/events/resource_ids.rs index bce90d032f..02dbc3b960 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs +++ b/crates/sqlbuild-python-output/src/result_output/_helpers/events/resource_ids.rs @@ -1,8 +1,8 @@ //! Canonical execution-protocol identities of SQL test cases and attached audits. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::pyfunction; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::nested_progress::main::audit_resource_id::audit_resource_id as audit_id; use sqlbuild_core::nested_progress::main::sql_test_resource_id::sql_test_resource_id as sql_test_id; @@ -34,7 +34,7 @@ pub(crate) fn sql_test_resource_id( /// The stable identity of one attached audit execution. #[pyfunction] #[pyo3(signature = (**options))] -fn audit_resource_id(options: Option<&Bound<'_, PyDict>>) -> PyResult { +pub(crate) fn audit_resource_id(options: Option<&Bound<'_, PyDict>>) -> PyResult { let request = Request::new("audit_resource_id", options)?; let target_kind: Option> = request .optional("attached_target_kind")? @@ -53,7 +53,7 @@ fn audit_resource_id(options: Option<&Bound<'_, PyDict>>) -> PyResult { ) } -fn audit_identity_of( +pub(crate) fn audit_identity_of( audit_name: &Bound<'_, PyAny>, kind: &Bound<'_, PyAny>, (target, column): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), @@ -66,43 +66,3 @@ fn audit_identity_of( (target.as_deref(), column.as_deref()), )) } - -/// `sql_test_resource_id` of one result or plan entry carrying `test_name_field` and case fields. -pub(crate) fn sql_test_identity( - result: &Bound<'_, PyAny>, - test_name_field: &str, -) -> PyResult { - sql_test_resource_id( - &native_field(result, test_name_field)?.extract::()?, - native_field(result, "source_path")?, - native_field(result, "block_index")?.extract()?, - native_field(result, "case_name")?.extract()?, - ) -} - -/// `audit_resource_id` of one audit result or plan entry, whose name is in `audit_name_field`. -pub(crate) fn audit_identity( - result: &Bound<'_, PyAny>, - audit_name_field: &str, -) -> PyResult { - let target_kind: Bound<'_, PyAny> = native_field(result, "attached_target_kind")?; - let kind: Bound<'_, PyAny> = if target_kind.is_none() { - native_field(result, "attachment_kind")? - } else { - target_kind - }; - audit_identity_of( - &native_field(result, audit_name_field)?, - &kind, - ( - &native_field(result, "attached_target_name")?, - &native_field(result, "attached_column_name")?, - ), - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(sql_test_resource_id, module)?)?; - module.add_function(wrap_pyfunction!(audit_resource_id, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-output/src/result_output/_helpers/events/terminal_index.rs b/crates/sqlbuild-python-output/src/result_output/_helpers/events/terminal_index.rs new file mode 100644 index 0000000000..d27166d06a --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/_helpers/events/terminal_index.rs @@ -0,0 +1,103 @@ +//! The invocation-scoped index of canonical lifecycle facts that result output projects from. + +use crate::result_output::models::TerminalEventIndex; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyRef, PyResult, Python}; +use pyo3::pyclass; +use pyo3::types::{PyInt, PyString}; + +use sqlbuild_core::terminal_events::models::{TerminalFact, TerminalIndex}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_base::records::main::registry::native_attr::native_attr; +use sqlbuild_python_base::records::types::NativeAttr; + +/// The module attribute holding the `ContextVar` of the installed index. +pub(crate) const CONTEXT_ATTRIBUTE: &str = "_terminal_event_index_context"; +pub(crate) const CONTEXT_NAME: &str = "sqlbuild_terminal_event_index"; + +#[derive(Debug)] +pub(crate) struct IndexState { + pub(crate) index: TerminalIndex, + pub(crate) events: Vec>, +} + +/// One claimed terminal with its immutable canonical publication sequence. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct TerminalEventClaim { + #[pyo3(get)] + pub(crate) terminal: Py, + #[pyo3(get)] + pub(crate) event_sequence: usize, +} + +/// `payload.get(key)` when it is a string. +fn payload_text(payload: &Bound<'_, PyAny>, key: &str) -> PyResult> { + let value: Bound<'_, PyAny> = mapping_get(payload, key)?; + if value.is_instance_of::() { + return Ok(Some(value.extract()?)); + } + Ok(None) +} + +/// The payload attempt number, when it is an `int` (or `bool`, which Python compares as one). +fn payload_attempt_number(payload: &Bound<'_, PyAny>) -> PyResult> { + let value: Bound<'_, PyAny> = mapping_get(payload, "attempt_number")?; + if !value.is_instance_of::() { + return Ok(None); + } + let Ok(number) = value.extract::() else { + return Ok(None); + }; + Ok(Some(number)) +} + +fn optional_text(event: &Bound<'_, PyAny>, name: &str) -> PyResult> { + native_field(event, name)?.extract() +} + +/// The facts of one lifecycle event that terminal indexing reads. +pub(crate) fn terminal_fact(event: &Bound<'_, PyAny>) -> PyResult { + let payload: Bound<'_, PyAny> = native_field(event, "payload")?; + Ok(TerminalFact { + event_id: native_field(event, "event_id")?.extract()?, + event_type: native_field(event, "event_type")?.extract()?, + resource_id: optional_text(event, "resource_id")?, + resource_name: payload_text(&payload, "resource_name")?, + resource_attempt_id: optional_text(event, "resource_attempt_id")?, + attempt_number: payload_attempt_number(&payload)?, + operation_name: payload_text(&payload, "operation_name")?, + }) +} + +pub(crate) fn context_variable(py: Python<'_>) -> PyResult> { + native_attr(py, NativeAttr::TerminalEventIndexContext) +} + +/// The terminal event index installed for this invocation. +pub(crate) fn current_index(py: Python<'_>) -> PyResult>> { + let installed: Bound<'_, PyAny> = context_variable(py)?.call_method0("get")?; + if installed.is_none() { + return Ok(None); + } + Ok(Some(installed.extract()?)) +} + +/// The installed terminal event index, as an owned reference. +pub(crate) fn current_index_object(py: Python<'_>) -> PyResult>> { + let installed: Bound<'_, PyAny> = context_variable(py)?.call_method0("get")?; + if installed.is_none() { + return Ok(None); + } + Ok(Some(installed.extract()?)) +} + +impl TerminalEventClaim { + pub(crate) fn terminal_event<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { + self.terminal.bind(py).clone() + } + + pub(crate) fn sequence(&self) -> usize { + self.event_sequence + } +} diff --git a/crates/sqlbuild-python-output/src/result_output/_helpers/mod.rs b/crates/sqlbuild-python-output/src/result_output/_helpers/mod.rs new file mode 100644 index 0000000000..a97f4a7045 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod events; +pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs b/crates/sqlbuild-python-output/src/result_output/_helpers/results/aggregation.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs rename to crates/sqlbuild-python-output/src/result_output/_helpers/results/aggregation.rs index 1e07fa2b8f..9c38b88447 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs +++ b/crates/sqlbuild-python-output/src/result_output/_helpers/results/aggregation.rs @@ -1,8 +1,8 @@ //! Build `BuildExecutionResult` with aggregate counts and overall status from typed results. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyDict; -use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction}; use sqlbuild_core::build_aggregate::main::aggregate_build_counts::aggregate_build_counts; use sqlbuild_core::build_aggregate::models::{BuildCounts, BuildResultFacts, ModelResultFacts}; use sqlbuild_host::entries::models::ExecutorHost; @@ -12,7 +12,7 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::result_output::projection::Projection; +use crate::result_output::_helpers::results::projection::Projection; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::Request; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -202,7 +202,7 @@ pub(crate) fn aggregated_build_result<'py>( /// Compute aggregate counts and overall build status. #[pyfunction] #[pyo3(signature = (**options))] -fn aggregate_build_result<'py>( +pub(crate) fn aggregate_build_result<'py>( options: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("aggregate_build_result", options)?; @@ -212,8 +212,3 @@ fn aggregate_build_result<'py>( .collect::>()?; aggregated_build_result(request.py(), §ions) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(aggregate_build_result, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs b/crates/sqlbuild-python-output/src/result_output/_helpers/results/assets.rs similarity index 99% rename from crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs rename to crates/sqlbuild-python-output/src/result_output/_helpers/results/assets.rs index f9b8acfd88..2dbf3ad9c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs +++ b/crates/sqlbuild-python-output/src/result_output/_helpers/results/assets.rs @@ -8,7 +8,8 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::result_output::projection::{Projection, rounded}; +use crate::result_output::_helpers::results::projection::Projection; +use crate::result_output::_helpers::results::projection::rounded; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs b/crates/sqlbuild-python-output/src/result_output/_helpers/results/checks.rs similarity index 98% rename from crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs rename to crates/sqlbuild-python-output/src/result_output/_helpers/results/checks.rs index 20dc6a7cc0..a3725a4cf2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs +++ b/crates/sqlbuild-python-output/src/result_output/_helpers/results/checks.rs @@ -9,8 +9,11 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::result_output::assets::{or_none, without_none}; -use crate::bindings::_helpers::result_output::projection::{Projection, ResultIdentity, rounded}; +use crate::result_output::_helpers::results::assets::or_none; +use crate::result_output::_helpers::results::assets::without_none; +use crate::result_output::_helpers::results::projection::Projection; +use crate::result_output::_helpers::results::projection::ResultIdentity; +use crate::result_output::_helpers::results::projection::rounded; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::scopes::main::reports::json_to_python::json_to_python; use sqlbuild_python_planner::plan_output::main::any::any; diff --git a/crates/sqlbuild-python-output/src/result_output/_helpers/results/documents.rs b/crates/sqlbuild-python-output/src/result_output/_helpers/results/documents.rs new file mode 100644 index 0000000000..b210fde040 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/_helpers/results/documents.rs @@ -0,0 +1,551 @@ +//! Execution-result JSON documents for build, seed, load, clone, test, audit and scenario commands. + +use crate::result_output::main::results::audit_execution_json::audit_execution_json; +use crate::result_output::main::results::audit_projection::audit_projection; +use crate::result_output::main::results::test_execution_json::test_execution_json; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_host::models::{ + AuditExecutionResult, BuildExecutionResult, BuildStatus, ExecutionStatus, + ScenarioLocalRunStatus, ScenarioRunResult, SqlTestExecutionResult, +}; + +use crate::result_output::_helpers::results::aggregation::RetainedBuildResults; +use crate::result_output::_helpers::results::assets::function_assets; +use crate::result_output::_helpers::results::assets::load_assets; +use crate::result_output::_helpers::results::assets::model_assets; +use crate::result_output::_helpers::results::assets::python_node_assets; +use crate::result_output::_helpers::results::assets::seed_assets; +use crate::result_output::_helpers::results::assets::without_none; +use crate::result_output::_helpers::results::checks::audit_checks; +use crate::result_output::_helpers::results::checks::python_checks; +use crate::result_output::_helpers::results::checks::scenario_check; +use crate::result_output::_helpers::results::checks::scenario_result; +use crate::result_output::_helpers::results::checks::sql_test_checks; +use crate::result_output::_helpers::results::projection::Projection; +use crate::result_output::_helpers::results::projection::rounded; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::compile_outputs::main::indented_json::indented_json; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::native_host::types::ReadResult; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_planner::plan_output::main::any::any; +use sqlbuild_python_planner::plan_output::main::dict::dict; +use sqlbuild_python_planner::plan_output::main::text::text; + +const JSON_VERSION: i64 = 1; +pub(crate) const FAILED: &str = "failed"; +const SUCCESS: &str = "success"; +const SKIPPED: &str = "skipped"; +pub(crate) const BUILD_OPTIONS: (&[&str], &[&str]) = ( + &["result", "plan"], + &[ + "python_node_results", + "python_check_results", + "command", + "run_id", + "cost", + ], +); +const DEGRADABLE_BUILD_OPTIONS: (&[&str], &[&str]) = ( + &["result", "plan", "execution_succeeded"], + &[ + "python_node_results", + "python_check_results", + "command", + "run_id", + "cost", + ], +); +pub(crate) const TYPED_BUILD_OPTIONS: (&[&str], &[&str]) = + (&["plan", "execution_succeeded"], BUILD_OPTIONS.1); +pub(crate) const SCENARIO_OPTIONS: (&[&str], &[&str]) = ( + &["results"], + &["local", "run_namespace", "namespace_source", "duration_ms"], +); +pub(crate) const SNAPSHOT_OPTIONS: (&[&str], &[&str]) = ( + &["results"], + &["refresh", "run_namespace", "namespace_source"], +); +/// The optional sections of a document after `checks`. +#[derive(Default)] +pub(crate) struct Extras<'py> { + pub(crate) scenarios: Vec>, + pub(crate) run_id: Option>, + pub(crate) cost: Option>, + pub(crate) execution: Option>, + /// Entries appended after every other section. + pub(crate) trailing: Vec<(&'static str, Bound<'py, PyAny>)>, +} + +/// The document body shared by every command. +pub(crate) struct Document<'py> { + pub(crate) command: String, + pub(crate) status: String, + pub(crate) summary: Bound<'py, PyDict>, + pub(crate) assets: Vec>, + pub(crate) checks: Vec>, +} + +/// `json.dumps(payload, indent=2) + "\n"`, failed when the lifecycle terminal failed. +pub(crate) fn emitted<'py>( + projection: &Projection<'py>, + document: Document<'py>, + extras: Extras<'py>, +) -> PyResult { + let py: Python<'py> = document.summary.py(); + let mut status: String = document.status; + if let Some(terminal) = projection.lifecycle_terminal()? + && text(&native_field(&terminal, "event_type")?)?.ends_with(FAILED) + { + status = FAILED.to_owned(); + } + let payload: Bound<'py, PyDict> = dict( + py, + vec![ + ("version", any(py, JSON_VERSION)?), + ("command", any(py, document.command)?), + ("status", any(py, status)?), + ("summary", document.summary.into_any()), + ("assets", PyTuple::new(py, document.assets)?.into_any()), + ("checks", PyTuple::new(py, document.checks)?.into_any()), + ], + )?; + if !extras.scenarios.is_empty() { + payload.set_item("scenarios", PyTuple::new(py, extras.scenarios)?)?; + } + if let Some(run_id) = extras.run_id.filter(|run_id| !run_id.is_none()) { + payload.set_item("run_id", run_id)?; + } + if let Some(cost) = extras.cost.filter(|cost| !cost.is_none()) { + payload.set_item("cost", cost)?; + } + if let Some(execution) = extras.execution { + payload.set_item("execution", execution)?; + } + for (key, value) in extras.trailing { + payload.set_item(key, value)?; + } + Ok(indented_json(payload.as_any())? + "\n") +} + +pub(crate) fn count_status( + results: &[Bound<'_, PyAny>], + field: &str, + status: &str, +) -> PyResult { + let mut count: usize = 0; + for result in results { + if native_field(&native_field(result, field)?, "value")?.eq(status)? { + count += 1; + } + } + Ok(count) +} + +fn count_truthy(results: &[Bound<'_, PyAny>], field: &str) -> PyResult { + let mut count: usize = 0; + for result in results { + if native_field(result, field)?.is_truthy()? { + count += 1; + } + } + Ok(count) +} + +pub(crate) fn summary<'py>( + py: Python<'py>, + entries: &[(&str, usize)], +) -> PyResult> { + dict( + py, + entries + .iter() + .map(|(key, count)| Ok((*key, any(py, *count)?))) + .collect::>()?, + ) +} + +pub(crate) fn outcome_status(fail_count: usize) -> String { + if fail_count == 0 { SUCCESS } else { FAILED }.to_owned() +} + +/// The argument when it was passed and is not `None`. +pub(crate) fn passed<'py>( + arguments: &KeywordArguments<'py>, + name: &str, +) -> PyResult>> { + let value: Bound<'py, PyAny> = arguments.value(name)?; + Ok((!value.is_none()).then_some(value)) +} + +/// A sequence argument that defaults to empty. +fn sequence<'py>(py: Python<'py>, value: Option>) -> Bound<'py, PyAny> { + value.unwrap_or_else(|| PyTuple::empty(py).into_any()) +} + +pub(crate) fn count(value: i64) -> PyResult { + usize::try_from(value).map_err(|_| { + pyo3::exceptions::PyOverflowError::new_err("can't convert negative int to unsigned") + }) +} + +/// The build document with its audit-history projection, or a minimal degraded document. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn format_degradable_build_execution_json<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "format_degradable_build_execution_json", + options, + DEGRADABLE_BUILD_OPTIONS, + )?; + let result = BuildExecutionResult::read(&arguments.value("result")?)?; + degradable_build_json(py, &arguments, &result) +} + +pub(crate) fn degradable_build_json<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, + result: &BuildExecutionResult>, +) -> PyResult { + let execution_succeeded: bool = arguments.flag("execution_succeeded")?; + let rendered: PyResult = audit_projection( + py, + ( + result.audit_result_projection_attempted_count, + result.audit_result_projection_written_count, + result.audit_result_projection_failed_count, + ), + ) + .and_then(|trailing| build_json(py, arguments, result, trailing)); + let error: pyo3::PyErr = match rendered { + Ok(payload) => return Ok(payload), + Err(error) => error, + }; + let degradable: bool = error.is_instance_of::(py) + || error.is_instance_of::(py) + || error.is_instance( + py, + &host_entry(py, ExecutorHost::ObservabilityValidationError)?, + ); + if !degradable { + return Err(error); + } + let command: String = if arguments.passed("command")? { + arguments.value("command")?.extract()? + } else { + "build".to_owned() + }; + degraded_document( + py, + (command, if execution_succeeded { SUCCESS } else { FAILED }), + error.value(py).as_any(), + execution_succeeded, + ) +} + +pub(crate) fn build_json<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, + result: &BuildExecutionResult>, + trailing: Vec<(&'static str, Bound<'py, PyAny>)>, +) -> PyResult { + let plan: Bound<'py, PyAny> = arguments.value("plan")?; + let command: String = if arguments.passed("command")? { + arguments.value("command")?.extract()? + } else { + "build".to_owned() + }; + let projection: Projection<'py> = Projection::installed(py)?; + let python_node_results: Bound<'py, PyAny> = + sequence(py, passed(arguments, "python_node_results")?); + let python_check_results: Bound<'py, PyAny> = + sequence(py, passed(arguments, "python_check_results")?); + let retained = RetainedBuildResults::new(result, &projection)?; + let nodes: Vec> = projection.terminal_results(&python_node_results)?; + let checks_run: Vec> = projection.terminal_results(&python_check_results)?; + let partial = retained.is_partial(result) + || nodes.len() != python_node_results.len()? + || checks_run.len() != python_check_results.len()?; + let python_fail: usize = count_status(&nodes, "status", FAILED)?; + let python_success: usize = count_status(&nodes, "status", SUCCESS)?; + let python_skipped: usize = count_status(&nodes, "status", SKIPPED)?; + let check_fail: usize = count_truthy(&checks_run, "failed")?; + let check_warn: usize = count_truthy(&checks_run, "warned")?; + let check_pass: usize = count_truthy(&checks_run, "passed")?; + let mut python_warnings: usize = 0; + for warned in nodes.iter().chain(&checks_run) { + python_warnings += native_field(warned, "warning_messages")?.len()?; + } + python_warnings += retained + .loads + .iter() + .map(|result| result.warning_messages.len()) + .sum::(); + let status: String = + if result.status == BuildStatus::Failed || python_fail > 0 || check_fail > 0 { + FAILED.to_owned() + } else { + SUCCESS.to_owned() + }; + let mut checks: Vec> = sql_test_checks(&projection, &retained.tests)?; + for audits in [&retained.source_audits, &retained.end_audits] { + checks.extend(audit_checks(&projection, audits, false)?); + } + for model in &retained.models { + checks.extend(audit_checks( + &projection, + &model.audit_results.iter().collect::>(), + projection.terminal_recorded(model)?, + )?); + } + checks.extend(python_checks(&projection, &checks_run)?); + let mut assets: Vec> = + model_assets(&projection, &retained.models, Some(&plan))?; + assets.extend(seed_assets(&projection, &retained.seeds, Some(&plan))?); + assets.extend(function_assets( + &projection, + &retained.functions, + Some(&plan), + )?); + assets.extend(load_assets(&projection, &retained.loads)?); + assets.extend(python_node_assets(&projection, &nodes)?); + let counted: [usize; 5] = if partial { + let counts = retained.counts(); + [ + counts.success, + counts.failure, + counts.timeout, + counts.skipped, + counts.warning, + ] + } else { + [ + count(result.success_count)?, + count(result.failure_count)?, + count(result.timeout_count)?, + count(result.skipped_count)?, + count(result.warning_count)?, + ] + }; + let summary: Bound<'py, PyDict> = summary( + py, + &[ + ("success_count", counted[0] + python_success), + ("failure_count", counted[1] + python_fail + check_fail), + ("timeout_count", counted[2]), + ("skipped_count", counted[3] + python_skipped), + ("warning_count", counted[4] + check_warn + python_warnings), + ("python_check_pass_count", check_pass), + ("python_check_warn_count", check_warn), + ("python_check_fail_count", check_fail), + ], + )?; + emitted( + &projection, + Document { + command, + status, + summary, + assets, + checks, + }, + Extras { + run_id: passed(arguments, "run_id")?, + cost: passed(arguments, "cost")?, + trailing, + ..Extras::default() + }, + ) +} + +/// Format test command execution results as JSON. +#[pyfunction] +#[pyo3(signature = (*, results))] +pub(crate) fn format_test_execution_json<'py>( + py: Python<'py>, + results: Bound<'py, PyAny>, +) -> PyResult { + let all = Vec::>::read(&results)?; + test_execution_json(py, &all) +} + +/// Format audit command execution results as JSON. +#[pyfunction] +#[pyo3(signature = (*, results, configured_concurrency = None, worker_count = None))] +pub(crate) fn format_audit_execution_json<'py>( + py: Python<'py>, + results: Bound<'py, PyAny>, + configured_concurrency: Option>, + worker_count: Option>, +) -> PyResult { + let all = Vec::>::read(&results)?; + audit_execution_json(py, &all, (configured_concurrency, worker_count), Vec::new()) +} + +pub(crate) fn namespace_execution<'py>( + py: Python<'py>, + run_namespace: Option>, + namespace_source: Bound<'py, PyAny>, +) -> PyResult> { + dict( + py, + vec![ + ( + "scenario_namespace", + run_namespace.unwrap_or_else(|| py.None().into_bound(py)), + ), + ("scenario_namespace_source", namespace_source), + ], + ) +} + +fn scenario_failed(result: &ScenarioRunResult>, local: bool) -> bool { + if !local { + return result.status == ExecutionStatus::Failed; + } + matches!( + result.local_status, + Some(ScenarioLocalRunStatus::Fail | ScenarioLocalRunStatus::Error) + ) +} + +/// Format scenario test command execution results as JSON. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn format_scenario_execution_json<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "format_scenario_execution_json", + options, + SCENARIO_OPTIONS, + )?; + let results = Vec::>::read(&arguments.value("results")?)?; + scenario_json(py, &arguments, &results) +} + +pub(crate) fn scenario_json<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, + results: &[ScenarioRunResult>], +) -> PyResult { + let local: bool = arguments.passed("local")? && arguments.flag("local")?; + let run_namespace: Option> = passed(arguments, "run_namespace")?; + let namespace_source: Option> = passed(arguments, "namespace_source")?; + let duration_ms: Option = arguments.value("duration_ms")?.extract()?; + let mut projection: Projection<'py> = Projection::installed(py)?; + let mut kept: Vec<&ScenarioRunResult>> = Vec::new(); + for result in results { + let name = &result.scenario_name; + if projection.resource_terminal_recorded(name, &format!("sql_scenario:{name}"))? { + kept.push(result); + } + } + let mut fail_count: usize = 0; + let (mut assets, mut checks, mut scenarios) = (Vec::new(), Vec::new(), Vec::new()); + for result in &kept { + fail_count += usize::from(scenario_failed(result, local)); + projection.scenario = Some(result.scenario_name.clone()); + assets.extend(seed_assets( + &projection, + &result.seed_results.iter().collect::>(), + None, + )?); + assets.extend(function_assets( + &projection, + &result.function_results.iter().collect::>(), + None, + )?); + assets.extend(model_assets( + &projection, + &result.model_results.iter().collect::>(), + None, + )?); + projection.scenario = None; + checks.push(scenario_check(py, result)?); + scenarios.push(scenario_result(py, result)?); + } + let mut summary: Bound<'py, PyDict> = summary( + py, + &[ + ("pass_count", kept.len() - fail_count), + ("fail_count", fail_count), + ("total_count", kept.len()), + ], + )?; + if let Some(duration) = duration_ms { + summary = without_none( + py, + vec![ + ("pass_count", any(py, kept.len() - fail_count)?), + ("fail_count", any(py, fail_count)?), + ("total_count", any(py, kept.len())?), + ("duration_ms", rounded(py, duration)?), + ], + )?; + } + emitted( + &projection, + Document { + command: "scenario test".to_owned(), + status: outcome_status(fail_count), + summary, + assets, + checks, + }, + Extras { + scenarios, + execution: Some(namespace_execution( + py, + run_namespace, + namespace_source.map_or_else(|| any(py, "unset"), Ok)?, + )?), + ..Extras::default() + }, + ) +} + +/// Warn about a failed result projection, then format a minimal valid document. +fn degraded_document( + py: Python<'_>, + (command, status): (String, &str), + error: &Bound<'_, PyAny>, + execution_succeeded: bool, +) -> PyResult { + let completion: &str = if execution_succeeded { + "warehouse execution completed successfully" + } else { + "execution result remains authoritative" + }; + let stderr: Bound<'_, PyAny> = py.import("sys")?.getattr("stderr")?; + stderr.call_method1( + "write", + (format!( + "warning: integration output degraded: {}; {completion}\n", + error.str()? + ),), + )?; + stderr.call_method0("flush")?; + let payload: Bound<'_, PyDict> = dict( + py, + vec![ + ("version", any(py, JSON_VERSION)?), + ("command", any(py, command)?), + ("status", any(py, status)?), + ("summary", PyDict::new(py).into_any()), + ("assets", PyList::empty(py).into_any()), + ("checks", PyList::empty(py).into_any()), + ("projection_degraded", any(py, true)?), + ], + )?; + Ok(indented_json(payload.as_any())? + "\n") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/getters.rs b/crates/sqlbuild-python-output/src/result_output/_helpers/results/getters.rs similarity index 66% rename from crates/sqlbuild-python/src/bindings/_helpers/result_output/getters.rs rename to crates/sqlbuild-python-output/src/result_output/_helpers/results/getters.rs index e48043447b..407d844707 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/getters.rs +++ b/crates/sqlbuild-python-output/src/result_output/_helpers/results/getters.rs @@ -3,34 +3,12 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::result_output::integration_envelope::{ - IntegrationAssetResult, IntegrationCheckResult, IntegrationResultEnvelope, -}; -use crate::bindings::_helpers::result_output::terminal_index::TerminalEventClaim; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::result_output::_helpers::events::integration_envelope::IntegrationAssetResult; +use crate::result_output::_helpers::events::integration_envelope::IntegrationCheckResult; +use crate::result_output::_helpers::events::integration_envelope::IntegrationResultEnvelope; +use crate::result_output::_helpers::events::terminal_index::TerminalEventClaim; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[ - GetterEntry { - class: class_of::, - read: read_integration_asset_result, - }, - GetterEntry { - class: class_of::, - read: read_integration_check_result, - }, - GetterEntry { - class: class_of::, - read: read_integration_result_envelope, - }, - GetterEntry { - class: class_of::, - read: read_terminal_event_claim, - }, -]; - -fn read_integration_asset_result<'py>( +pub(crate) fn read_integration_asset_result<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -47,7 +25,7 @@ fn read_integration_asset_result<'py>( }) } -fn read_integration_check_result<'py>( +pub(crate) fn read_integration_check_result<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -64,7 +42,7 @@ fn read_integration_check_result<'py>( }) } -fn read_integration_result_envelope<'py>( +pub(crate) fn read_integration_result_envelope<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -85,7 +63,7 @@ fn read_integration_result_envelope<'py>( }) } -fn read_terminal_event_claim<'py>( +pub(crate) fn read_terminal_event_claim<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-output/src/result_output/_helpers/results/mod.rs b/crates/sqlbuild-python-output/src/result_output/_helpers/results/mod.rs new file mode 100644 index 0000000000..a4dbac80da --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/_helpers/results/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod aggregation; +pub(crate) mod assets; +pub(crate) mod checks; +pub(crate) mod documents; +pub(crate) mod getters; +pub(crate) mod projection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs b/crates/sqlbuild-python-output/src/result_output/_helpers/results/projection.rs similarity index 96% rename from crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs rename to crates/sqlbuild-python-output/src/result_output/_helpers/results/projection.rs index d9228df633..7f5094ad9d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs +++ b/crates/sqlbuild-python-output/src/result_output/_helpers/results/projection.rs @@ -1,5 +1,7 @@ //! Executor results matched to the canonical terminals of the installed event index. +use crate::result_output::main::events::sql_test_identity::sql_test_identity; +use crate::result_output::main::results::audit_check_id::audit_check_id; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; use pyo3::types::{PyFloat, PyInt}; use sqlbuild_core::nested_progress::main::audit_resource_id::audit_resource_id; @@ -11,8 +13,8 @@ use sqlbuild_host::models::{ }; use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::result_output::resource_ids::{audit_identity, sql_test_identity}; -use crate::bindings::_helpers::result_output::terminal_index::{TerminalEventIndex, current_index}; +use crate::result_output::_helpers::events::terminal_index::current_index; +use crate::result_output::models::TerminalEventIndex; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_planner::plan_output::main::text::text; @@ -65,11 +67,6 @@ pub(crate) fn sql_test_check_id(result: &Bound<'_, PyAny>) -> PyResult { sql_test_identity(result, "test_name") } -/// `audit_resource_id` of one audit result. -pub(crate) fn audit_check_id(result: &Bound<'_, PyAny>) -> PyResult { - audit_identity(result, "audit_name") -} - /// The canonical resource name and identifier of an executor result. pub(crate) fn resource_identity(result: &Bound<'_, PyAny>) -> PyResult> { let name = |field: &str| -> PyResult { text(&native_field(result, field)?) }; diff --git a/crates/sqlbuild-python-output/src/result_output/constants.rs b/crates/sqlbuild-python-output/src/result_output/constants.rs new file mode 100644 index 0000000000..3d0e262057 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/constants.rs @@ -0,0 +1,34 @@ +//! Public constants of result output. + +use crate::result_output::_helpers::events::integration_envelope::IntegrationAssetResult; +use crate::result_output::_helpers::events::integration_envelope::IntegrationCheckResult; +use crate::result_output::_helpers::events::integration_envelope::IntegrationResultEnvelope; +use crate::result_output::_helpers::events::terminal_index::TerminalEventClaim; +use crate::result_output::_helpers::results::getters::read_integration_asset_result; +use crate::result_output::_helpers::results::getters::read_integration_check_result; +use crate::result_output::_helpers::results::getters::read_integration_result_envelope; +use crate::result_output::_helpers::results::getters::read_terminal_event_claim; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +pub const RESULT_PATH_ENV: &str = "SQLBUILD_INTEGRATION_RESULT_PATH"; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_integration_asset_result, + }, + GetterEntry { + class: class_of::, + read: read_integration_check_result, + }, + GetterEntry { + class: class_of::, + read: read_integration_result_envelope, + }, + GetterEntry { + class: class_of::, + read: read_terminal_event_claim, + }, +]; diff --git a/crates/sqlbuild-python-output/src/result_output/main/events/audit_identity.rs b/crates/sqlbuild-python-output/src/result_output/main/events/audit_identity.rs new file mode 100644 index 0000000000..87b477b6bb --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/events/audit_identity.rs @@ -0,0 +1,23 @@ +//! `audit_resource_id` of one audit result or plan entry, whose name is in `audit_name_field`. + +use crate::result_output::_helpers::events::resource_ids::audit_identity_of; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// `audit_resource_id` of one audit result or plan entry, whose name is in `audit_name_field`. +pub fn audit_identity(result: &Bound<'_, PyAny>, audit_name_field: &str) -> PyResult { + let target_kind: Bound<'_, PyAny> = native_field(result, "attached_target_kind")?; + let kind: Bound<'_, PyAny> = if target_kind.is_none() { + native_field(result, "attachment_kind")? + } else { + target_kind + }; + audit_identity_of( + &native_field(result, audit_name_field)?, + &kind, + ( + &native_field(result, "attached_target_name")?, + &native_field(result, "attached_column_name")?, + ), + ) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/events/current_terminal_event_index.rs b/crates/sqlbuild-python-output/src/result_output/main/events/current_terminal_event_index.rs new file mode 100644 index 0000000000..fd2f480dd8 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/events/current_terminal_event_index.rs @@ -0,0 +1,11 @@ +//! Return the terminal event index installed for this invocation. + +use crate::result_output::_helpers::events::terminal_index::context_variable; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; + +/// Return the terminal event index installed for this invocation. +#[pyfunction] +pub fn current_terminal_event_index(py: Python<'_>) -> PyResult> { + context_variable(py)?.call_method0("get") +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/events/mod.rs b/crates/sqlbuild-python-output/src/result_output/main/events/mod.rs new file mode 100644 index 0000000000..ff32884ba4 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/events/mod.rs @@ -0,0 +1,8 @@ +pub mod audit_identity; +pub mod current_terminal_event_index; +pub mod register_event_writer; +pub mod register_integration_envelope; +pub mod register_resource_ids; +pub mod register_terminal_index; +pub mod sql_test_identity; +pub mod terminal_event_index_scope; diff --git a/crates/sqlbuild-python-output/src/result_output/main/events/register_event_writer.rs b/crates/sqlbuild-python-output/src/result_output/main/events/register_event_writer.rs new file mode 100644 index 0000000000..02b1f8226e --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/events/register_event_writer.rs @@ -0,0 +1,9 @@ +//! Register the event writer functions and classes. + +use crate::result_output::models::ExecutionEventWriter; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the event writer functions and classes. +pub fn register_event_writer(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/events/register_integration_envelope.rs b/crates/sqlbuild-python-output/src/result_output/main/events/register_integration_envelope.rs new file mode 100644 index 0000000000..5bf0958d98 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/events/register_integration_envelope.rs @@ -0,0 +1,13 @@ +//! Register the integration envelope functions and classes. + +use crate::result_output::_helpers::events::integration_envelope::IntegrationAssetResult; +use crate::result_output::_helpers::events::integration_envelope::IntegrationCheckResult; +use crate::result_output::_helpers::events::integration_envelope::IntegrationResultEnvelope; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +/// Register the integration envelope functions and classes. +pub fn register_integration_envelope(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::() +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/events/register_resource_ids.rs b/crates/sqlbuild-python-output/src/result_output/main/events/register_resource_ids.rs new file mode 100644 index 0000000000..a155ef56de --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/events/register_resource_ids.rs @@ -0,0 +1,13 @@ +//! Register the resource ids functions and classes. + +use crate::result_output::_helpers::events::resource_ids::audit_resource_id; +use crate::result_output::_helpers::events::resource_ids::sql_test_resource_id; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the resource ids functions and classes. +pub fn register_resource_ids(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(sql_test_resource_id, module)?)?; + module.add_function(wrap_pyfunction!(audit_resource_id, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/events/register_terminal_index.rs b/crates/sqlbuild-python-output/src/result_output/main/events/register_terminal_index.rs new file mode 100644 index 0000000000..1cdc972a9e --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/events/register_terminal_index.rs @@ -0,0 +1,30 @@ +//! Register the terminal index functions and classes. + +use crate::result_output::_helpers::events::terminal_index::CONTEXT_ATTRIBUTE; +use crate::result_output::_helpers::events::terminal_index::CONTEXT_NAME; +use crate::result_output::_helpers::events::terminal_index::TerminalEventClaim; +use crate::result_output::main::events::current_terminal_event_index::current_terminal_event_index; +use crate::result_output::main::events::terminal_event_index_scope::terminal_event_index_scope; +use crate::result_output::models::TerminalEventIndex; +use crate::result_output::models::TerminalEventIndexScope; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::wrap_pyfunction; + +/// Register the terminal index functions and classes. +pub fn register_terminal_index(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py: Python<'_> = module.py(); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + pyo3::types::PyDictMethods::set_item(&keywords, "default", py.None())?; + module.add( + CONTEXT_ATTRIBUTE, + py.import("contextvars")? + .getattr("ContextVar")? + .call((CONTEXT_NAME,), Some(&keywords))?, + )?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(current_terminal_event_index, module)?)?; + module.add_function(wrap_pyfunction!(terminal_event_index_scope, module)?) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/events/sql_test_identity.rs b/crates/sqlbuild-python-output/src/result_output/main/events/sql_test_identity.rs new file mode 100644 index 0000000000..cf8dbc8525 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/events/sql_test_identity.rs @@ -0,0 +1,15 @@ +//! `sql_test_resource_id` of one result or plan entry carrying `test_name_field` and case fields. + +use crate::result_output::_helpers::events::resource_ids::sql_test_resource_id; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// `sql_test_resource_id` of one result or plan entry carrying `test_name_field` and case fields. +pub fn sql_test_identity(result: &Bound<'_, PyAny>, test_name_field: &str) -> PyResult { + sql_test_resource_id( + &native_field(result, test_name_field)?.extract::()?, + native_field(result, "source_path")?, + native_field(result, "block_index")?.extract()?, + native_field(result, "case_name")?.extract()?, + ) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/events/terminal_event_index_scope.rs b/crates/sqlbuild-python-output/src/result_output/main/events/terminal_event_index_scope.rs new file mode 100644 index 0000000000..4ee7e1a1ce --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/events/terminal_event_index_scope.rs @@ -0,0 +1,16 @@ +//! Install and reliably restore one invocation terminal event index. + +use crate::result_output::models::TerminalEventIndex; +use crate::result_output::models::TerminalEventIndexScope; +use pyo3::prelude::Py; +use pyo3::pyfunction; +use std::sync::Mutex; + +/// Install and reliably restore one invocation terminal event index. +#[pyfunction] +pub fn terminal_event_index_scope(index: Py) -> TerminalEventIndexScope { + TerminalEventIndexScope { + index, + token: Mutex::new(None), + } +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/mod.rs b/crates/sqlbuild-python-output/src/result_output/main/mod.rs new file mode 100644 index 0000000000..10a1888cfb --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/mod.rs @@ -0,0 +1,2 @@ +pub mod events; +pub mod results; diff --git a/crates/sqlbuild-python-output/src/result_output/main/results/audit_check_id.rs b/crates/sqlbuild-python-output/src/result_output/main/results/audit_check_id.rs new file mode 100644 index 0000000000..4cfb59f2ac --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/results/audit_check_id.rs @@ -0,0 +1,9 @@ +//! `audit_resource_id` of one audit result. + +use crate::result_output::main::events::audit_identity::audit_identity; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// `audit_resource_id` of one audit result. +pub fn audit_check_id(result: &Bound<'_, PyAny>) -> PyResult { + audit_identity(result, "audit_name") +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/results/audit_execution_json.rs b/crates/sqlbuild-python-output/src/result_output/main/results/audit_execution_json.rs new file mode 100644 index 0000000000..e9f5ddb439 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/results/audit_execution_json.rs @@ -0,0 +1,78 @@ +//! Audit reporting from already-converted executor results, before the document is serialized. + +use crate::result_output::_helpers::results::checks::audit_checks; +use crate::result_output::_helpers::results::documents::Document; +use crate::result_output::_helpers::results::documents::Extras; +use crate::result_output::_helpers::results::documents::emitted; +use crate::result_output::_helpers::results::documents::outcome_status; +use crate::result_output::_helpers::results::documents::summary; +use crate::result_output::_helpers::results::projection::Projection; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_host::models::{AuditExecutionResult, AuditOutcome}; +use sqlbuild_python_planner::plan_output::main::any::any; +use sqlbuild_python_planner::plan_output::main::dict::dict; + +/// Audit reporting from already-converted executor results, before the document is serialized. +pub fn audit_execution_json<'py>( + py: Python<'py>, + all: &[AuditExecutionResult>], + (configured_concurrency, worker_count): (Option>, Option>), + trailing: Vec<(&'static str, Bound<'py, PyAny>)>, +) -> PyResult { + let configured_concurrency: Bound<'py, PyAny> = + configured_concurrency.map_or_else(|| any(py, 1_i64), Ok)?; + let projection: Projection<'py> = Projection::installed(py)?; + let results = projection.retained(all)?; + let fail_count = results + .iter() + .filter(|result| result.outcome == AuditOutcome::Error) + .count(); + let workers: Bound<'py, PyAny> = match worker_count.filter(|count| !count.is_none()) { + Some(workers) => workers, + None => py + .import("builtins")? + .getattr("min")? + .call1((&configured_concurrency, results.len()))?, + }; + emitted( + &projection, + Document { + command: "audit".to_owned(), + status: outcome_status(fail_count), + summary: summary( + py, + &[ + ( + "pass_count", + results + .iter() + .filter(|result| result.outcome == AuditOutcome::Pass) + .count(), + ), + ( + "warn_count", + results + .iter() + .filter(|result| result.outcome == AuditOutcome::Warn) + .count(), + ), + ("fail_count", fail_count), + ("total_count", results.len()), + ], + )?, + assets: Vec::new(), + checks: audit_checks(&projection, &results, false)?, + }, + Extras { + execution: Some(dict( + py, + vec![ + ("configured_concurrency", configured_concurrency), + ("worker_count", workers), + ], + )?), + trailing, + ..Extras::default() + }, + ) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/results/audit_projection.rs b/crates/sqlbuild-python-output/src/result_output/main/results/audit_projection.rs new file mode 100644 index 0000000000..4f88c8b782 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/results/audit_projection.rs @@ -0,0 +1,37 @@ +//! The audit-history projection counts, flagged as degraded when any write failed. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyList; +use sqlbuild_python_planner::plan_output::main::any::any; +use sqlbuild_python_planner::plan_output::main::dict::dict; + +/// The audit-history projection counts, flagged as degraded when any write failed. +pub fn audit_projection<'py>( + py: Python<'py>, + (attempted, written, failed): (i64, i64, i64), +) -> PyResult)>> { + let counts: Vec<(&str, Bound<'py, PyAny>)> = [ + ("attempted_count", attempted), + ("written_count", written), + ("failed_count", failed), + ] + .into_iter() + .map(|(key, value)| Ok((key, any(py, value)?))) + .collect::>()?; + let failed: bool = counts[2].1.is_truthy()?; + let mut entries: Vec<(&'static str, Bound<'py, PyAny>)> = vec![( + "audit_result_projection", + dict(py, counts.clone())?.into_any(), + )]; + if failed { + let mut reason: Vec<(&str, Bound<'py, PyAny>)> = + vec![("reason", any(py, "audit_result_projection_failure")?)]; + reason.extend(counts); + entries.push(("projection_degraded", any(py, true)?)); + entries.push(( + "projection_degradation_reasons", + PyList::new(py, [dict(py, reason)?])?.into_any(), + )); + } + Ok(entries) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/results/format_build_execution_json.rs b/crates/sqlbuild-python-output/src/result_output/main/results/format_build_execution_json.rs new file mode 100644 index 0000000000..24e60b4f95 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/results/format_build_execution_json.rs @@ -0,0 +1,23 @@ +//! Format build command execution results as JSON. + +use crate::result_output::_helpers::results::documents::BUILD_OPTIONS; +use crate::result_output::_helpers::results::documents::build_json; +use pyo3::prelude::{Bound, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_host::models::BuildExecutionResult; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::types::ReadResult; + +/// Format build command execution results as JSON. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn format_build_execution_json<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "format_build_execution_json", options, BUILD_OPTIONS)?; + let result = BuildExecutionResult::read(&arguments.value("result")?)?; + build_json(py, &arguments, &result, Vec::new()) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/results/format_build_execution_json_with.rs b/crates/sqlbuild-python-output/src/result_output/main/results/format_build_execution_json_with.rs new file mode 100644 index 0000000000..0acf152fa6 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/results/format_build_execution_json_with.rs @@ -0,0 +1,23 @@ +//! Production build completion already owns the converted executor result. + +use crate::result_output::_helpers::results::documents::TYPED_BUILD_OPTIONS; +use crate::result_output::_helpers::results::documents::degradable_build_json; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_host::models::BuildExecutionResult; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// Production build completion already owns the converted executor result. +pub fn format_build_execution_json_with<'py>( + py: Python<'py>, + result: &BuildExecutionResult>, + options: Option>, +) -> PyResult { + let arguments = KeywordArguments::read( + py, + "format_build_execution_json", + options, + TYPED_BUILD_OPTIONS, + )?; + degradable_build_json(py, &arguments, result) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/results/format_clone_execution_json.rs b/crates/sqlbuild-python-output/src/result_output/main/results/format_clone_execution_json.rs new file mode 100644 index 0000000000..204dbee3d1 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/results/format_clone_execution_json.rs @@ -0,0 +1,67 @@ +//! Format direct clone command execution results as JSON. + +use crate::result_output::_helpers::results::assets::clone_asset; +use crate::result_output::_helpers::results::documents::Document; +use crate::result_output::_helpers::results::documents::Extras; +use crate::result_output::_helpers::results::documents::FAILED; +use crate::result_output::_helpers::results::documents::emitted; +use crate::result_output::_helpers::results::documents::outcome_status; +use crate::result_output::_helpers::results::documents::summary; +use crate::result_output::_helpers::results::projection::Projection; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_planner::plan_output::main::items::items; +use sqlbuild_python_planner::plan_output::main::text::text; + +/// Format direct clone command execution results as JSON. +#[pyfunction] +#[pyo3(signature = (*, result, resource_types_by_name, phase_timings = None))] +pub fn format_clone_execution_json<'py>( + py: Python<'py>, + result: Bound<'py, PyAny>, + resource_types_by_name: Bound<'py, PyAny>, + phase_timings: Option>, +) -> PyResult { + let projection: Projection<'py> = Projection::installed(py)?; + let mut assets: Vec> = Vec::new(); + let (mut failure_count, mut warning_count): (usize, usize) = (0, 0); + for item in items(&native_field(&result, "item_results")?)? { + let name: String = text(&native_field(&item, "name")?)?; + let resource_type: String = text(&resource_types_by_name.get_item(&name)?)?; + if !projection.resource_terminal_recorded(&name, &format!("{resource_type}:{name}"))? { + continue; + } + let status: Bound<'py, PyAny> = native_field(&native_field(&item, "status")?, "value")?; + failure_count += usize::from(status.eq(FAILED)?); + warning_count += usize::from(status.eq("warning")?); + assets.push(clone_asset(&projection, &item, &resource_type)?); + } + let total: usize = assets.len(); + emitted( + &projection, + Document { + command: "clone".to_owned(), + status: outcome_status(failure_count), + summary: summary( + py, + &[ + ("success_count", total - failure_count - warning_count), + ("failure_count", failure_count), + ("warning_count", warning_count), + ("total_count", total), + ], + )?, + assets, + checks: Vec::new(), + }, + Extras { + trailing: phase_timings + .into_iter() + .map(|timings| ("phase_timings", timings.into_any())) + .collect(), + ..Extras::default() + }, + ) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/results/format_load_execution_json.rs b/crates/sqlbuild-python-output/src/result_output/main/results/format_load_execution_json.rs new file mode 100644 index 0000000000..ae05f3d35d --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/results/format_load_execution_json.rs @@ -0,0 +1,60 @@ +//! Format load command execution results as JSON. + +use crate::result_output::_helpers::results::assets::load_assets; +use crate::result_output::_helpers::results::documents::Document; +use crate::result_output::_helpers::results::documents::Extras; +use crate::result_output::_helpers::results::documents::emitted; +use crate::result_output::_helpers::results::documents::outcome_status; +use crate::result_output::_helpers::results::documents::summary; +use crate::result_output::_helpers::results::projection::Projection; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_host::models::{ExecutionStatus, LoadExecutionResult}; +use sqlbuild_python_base::native_host::types::ReadResult; + +/// Format load command execution results as JSON. +#[pyfunction] +#[pyo3(signature = (*, results))] +pub fn format_load_execution_json<'py>( + py: Python<'py>, + results: Bound<'py, PyAny>, +) -> PyResult { + let projection: Projection<'py> = Projection::installed(py)?; + let all = Vec::>::read(&results)?; + let results = projection.retained(&all)?; + let fail_count = results + .iter() + .filter(|result| result.status == ExecutionStatus::Failed) + .count(); + emitted( + &projection, + Document { + command: "load".to_owned(), + status: outcome_status(fail_count), + summary: summary( + py, + &[ + ( + "success_count", + results + .iter() + .filter(|result| result.status == ExecutionStatus::Success) + .count(), + ), + ("failure_count", fail_count), + ( + "skipped_count", + results + .iter() + .filter(|result| result.status == ExecutionStatus::Skipped) + .count(), + ), + ("total_count", results.len()), + ], + )?, + assets: load_assets(&projection, &results)?, + checks: Vec::new(), + }, + Extras::default(), + ) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/results/format_scenario_execution_json_with.rs b/crates/sqlbuild-python-output/src/result_output/main/results/format_scenario_execution_json_with.rs new file mode 100644 index 0000000000..2f30bce924 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/results/format_scenario_execution_json_with.rs @@ -0,0 +1,23 @@ +//! Production scenario completion already owns its converted executor results. + +use crate::result_output::_helpers::results::documents::SCENARIO_OPTIONS; +use crate::result_output::_helpers::results::documents::scenario_json; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyDict; +use sqlbuild_host::models::ScenarioRunResult; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +/// Production scenario completion already owns its converted executor results. +pub fn format_scenario_execution_json_with<'py>( + py: Python<'py>, + results: &[ScenarioRunResult>], + options: Option>, +) -> PyResult { + let arguments = KeywordArguments::read( + py, + "format_scenario_execution_json", + options, + (&[], SCENARIO_OPTIONS.1), + )?; + scenario_json(py, &arguments, results) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/results/format_scenario_snapshot_execution_json.rs b/crates/sqlbuild-python-output/src/result_output/main/results/format_scenario_snapshot_execution_json.rs new file mode 100644 index 0000000000..33c0b02ebc --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/results/format_scenario_snapshot_execution_json.rs @@ -0,0 +1,76 @@ +//! Format scenario snapshot sync/refresh execution results as JSON. + +use crate::result_output::_helpers::results::checks::snapshot_result; +use crate::result_output::_helpers::results::documents::Document; +use crate::result_output::_helpers::results::documents::Extras; +use crate::result_output::_helpers::results::documents::FAILED; +use crate::result_output::_helpers::results::documents::SNAPSHOT_OPTIONS; +use crate::result_output::_helpers::results::documents::count_status; +use crate::result_output::_helpers::results::documents::emitted; +use crate::result_output::_helpers::results::documents::namespace_execution; +use crate::result_output::_helpers::results::documents::outcome_status; +use crate::result_output::_helpers::results::documents::passed; +use crate::result_output::_helpers::results::documents::summary; +use crate::result_output::_helpers::results::projection::Projection; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_planner::plan_output::main::any::any; +use sqlbuild_python_planner::plan_output::main::items::items; + +/// Format scenario snapshot sync/refresh execution results as JSON. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn format_scenario_snapshot_execution_json<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "format_scenario_snapshot_execution_json", + options, + SNAPSHOT_OPTIONS, + )?; + let results: Bound<'py, PyAny> = arguments.value("results")?; + let refresh: bool = arguments.passed("refresh")? && arguments.flag("refresh")?; + let run_namespace: Option> = passed(&arguments, "run_namespace")?; + let namespace_source: Option> = passed(&arguments, "namespace_source")?; + let projection: Projection<'py> = Projection::installed(py)?; + let results: Vec> = items(&results)?; + let fail_count: usize = count_status(&results, "status", FAILED)?; + emitted( + &projection, + Document { + command: if refresh { + "scenario snapshot refresh" + } else { + "scenario snapshot sync" + } + .to_owned(), + status: outcome_status(fail_count), + summary: summary( + py, + &[ + ("pass_count", results.len() - fail_count), + ("fail_count", fail_count), + ("total_count", results.len()), + ], + )?, + assets: Vec::new(), + checks: Vec::new(), + }, + Extras { + scenarios: results + .iter() + .map(snapshot_result) + .collect::>()?, + execution: Some(namespace_execution( + py, + run_namespace, + namespace_source.map_or_else(|| any(py, "unset"), Ok)?, + )?), + ..Extras::default() + }, + ) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/results/format_seed_execution_json.rs b/crates/sqlbuild-python-output/src/result_output/main/results/format_seed_execution_json.rs new file mode 100644 index 0000000000..79d8001cf3 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/results/format_seed_execution_json.rs @@ -0,0 +1,54 @@ +//! Format seed command execution results as JSON. + +use crate::result_output::_helpers::results::assets::seed_assets; +use crate::result_output::_helpers::results::documents::Document; +use crate::result_output::_helpers::results::documents::Extras; +use crate::result_output::_helpers::results::documents::emitted; +use crate::result_output::_helpers::results::documents::outcome_status; +use crate::result_output::_helpers::results::documents::summary; +use crate::result_output::_helpers::results::projection::Projection; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::pyfunction; +use sqlbuild_host::models::{ExecutionStatus, SeedExecutionResult}; +use sqlbuild_python_base::native_host::types::ReadResult; + +/// Format seed command execution results as JSON. +#[pyfunction] +#[pyo3(signature = (*, results, plan))] +pub fn format_seed_execution_json<'py>( + py: Python<'py>, + results: Bound<'py, PyAny>, + plan: Bound<'py, PyAny>, +) -> PyResult { + let projection: Projection<'py> = Projection::installed(py)?; + let all = Vec::::read(&results)?; + let results = projection.retained(&all)?; + let fail_count = results + .iter() + .filter(|result| result.status == ExecutionStatus::Failed) + .count(); + emitted( + &projection, + Document { + command: "seed".to_owned(), + status: outcome_status(fail_count), + summary: summary( + py, + &[ + ( + "success_count", + results + .iter() + .filter(|result| result.status == ExecutionStatus::Success) + .count(), + ), + ("failure_count", fail_count), + ("total_count", results.len()), + ], + )?, + assets: seed_assets(&projection, &results, Some(&plan))?, + checks: Vec::new(), + }, + Extras::default(), + ) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/results/mod.rs b/crates/sqlbuild-python-output/src/result_output/main/results/mod.rs new file mode 100644 index 0000000000..abfe754ebc --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/results/mod.rs @@ -0,0 +1,14 @@ +pub mod audit_check_id; +pub mod audit_execution_json; +pub mod audit_projection; +pub mod format_build_execution_json; +pub mod format_build_execution_json_with; +pub mod format_clone_execution_json; +pub mod format_load_execution_json; +pub mod format_scenario_execution_json_with; +pub mod format_scenario_snapshot_execution_json; +pub mod format_seed_execution_json; +pub mod register_aggregation; +pub mod register_documents; +pub mod test_execution_json; +pub mod write_execution_json_output; diff --git a/crates/sqlbuild-python-output/src/result_output/main/results/register_aggregation.rs b/crates/sqlbuild-python-output/src/result_output/main/results/register_aggregation.rs new file mode 100644 index 0000000000..04c78b7321 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/results/register_aggregation.rs @@ -0,0 +1,11 @@ +//! Register the aggregation functions and classes. + +use crate::result_output::_helpers::results::aggregation::aggregate_build_result; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the aggregation functions and classes. +pub fn register_aggregation(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(aggregate_build_result, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/results/register_documents.rs b/crates/sqlbuild-python-output/src/result_output/main/results/register_documents.rs new file mode 100644 index 0000000000..9686cc1ac8 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/results/register_documents.rs @@ -0,0 +1,34 @@ +//! Register the documents functions and classes. + +use crate::result_output::_helpers::results::documents::format_audit_execution_json; +use crate::result_output::_helpers::results::documents::format_degradable_build_execution_json; +use crate::result_output::_helpers::results::documents::format_scenario_execution_json; +use crate::result_output::_helpers::results::documents::format_test_execution_json; +use crate::result_output::main::results::format_build_execution_json::format_build_execution_json; +use crate::result_output::main::results::format_clone_execution_json::format_clone_execution_json; +use crate::result_output::main::results::format_load_execution_json::format_load_execution_json; +use crate::result_output::main::results::format_scenario_snapshot_execution_json::format_scenario_snapshot_execution_json; +use crate::result_output::main::results::format_seed_execution_json::format_seed_execution_json; +use crate::result_output::main::results::write_execution_json_output::write_execution_json_output; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +/// Register the documents functions and classes. +pub fn register_documents(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_build_execution_json, module)?)?; + module.add_function(wrap_pyfunction!(format_seed_execution_json, module)?)?; + module.add_function(wrap_pyfunction!(format_load_execution_json, module)?)?; + module.add_function(wrap_pyfunction!(format_clone_execution_json, module)?)?; + module.add_function(wrap_pyfunction!(format_test_execution_json, module)?)?; + module.add_function(wrap_pyfunction!(format_audit_execution_json, module)?)?; + module.add_function(wrap_pyfunction!(format_scenario_execution_json, module)?)?; + module.add_function(wrap_pyfunction!( + format_scenario_snapshot_execution_json, + module + )?)?; + module.add_function(wrap_pyfunction!( + format_degradable_build_execution_json, + module + )?)?; + module.add_function(wrap_pyfunction!(write_execution_json_output, module)?) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/results/test_execution_json.rs b/crates/sqlbuild-python-output/src/result_output/main/results/test_execution_json.rs new file mode 100644 index 0000000000..8a91b2a95f --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/results/test_execution_json.rs @@ -0,0 +1,48 @@ +//! The test command's document from the executor's already-converted results. + +use crate::result_output::_helpers::results::checks::sql_test_checks; +use crate::result_output::_helpers::results::documents::Document; +use crate::result_output::_helpers::results::documents::Extras; +use crate::result_output::_helpers::results::documents::emitted; +use crate::result_output::_helpers::results::documents::outcome_status; +use crate::result_output::_helpers::results::documents::summary; +use crate::result_output::_helpers::results::projection::Projection; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_host::models::{SqlTestExecutionResult, SqlTestOutcome}; + +/// The test command's document from the executor's already-converted results. +pub fn test_execution_json<'py>( + py: Python<'py>, + all: &[SqlTestExecutionResult>], +) -> PyResult { + let projection: Projection<'py> = Projection::installed(py)?; + let results = projection.retained(all)?; + let pass_count = results + .iter() + .filter(|result| result.outcome == SqlTestOutcome::Pass) + .count(); + let timeout_count = results + .iter() + .filter(|result| result.outcome == SqlTestOutcome::Timeout) + .count(); + let fail_count: usize = results.len() - pass_count; + emitted( + &projection, + Document { + command: "test".to_owned(), + status: outcome_status(fail_count), + summary: summary( + py, + &[ + ("pass_count", pass_count), + ("fail_count", fail_count), + ("timeout_count", timeout_count), + ("total_count", results.len()), + ], + )?, + assets: Vec::new(), + checks: sql_test_checks(&projection, &results)?, + }, + Extras::default(), + ) +} diff --git a/crates/sqlbuild-python-output/src/result_output/main/results/write_execution_json_output.rs b/crates/sqlbuild-python-output/src/result_output/main/results/write_execution_json_output.rs new file mode 100644 index 0000000000..76c9dd83c7 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/main/results/write_execution_json_output.rs @@ -0,0 +1,33 @@ +//! Write execution JSON to stdout or a requested side-channel file. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Write execution JSON to stdout or a requested side-channel file. +#[pyfunction] +#[pyo3(signature = (*, payload, json_output, json_output_path))] +pub fn write_execution_json_output( + py: Python<'_>, + payload: Bound<'_, PyAny>, + json_output: bool, + json_output_path: Bound<'_, PyAny>, +) -> PyResult<()> { + if !json_output_path.is_none() { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("parents", true)?; + keywords.set_item("exist_ok", true)?; + native_field(&json_output_path, "parent")?.call_method("mkdir", (), Some(&keywords))?; + let encoding: Bound<'_, PyDict> = PyDict::new(py); + encoding.set_item("encoding", "utf-8")?; + json_output_path.call_method("write_text", (payload,), Some(&encoding))?; + return Ok(()); + } + if json_output { + let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; + stdout.call_method1("write", (payload,))?; + stdout.call_method0("flush")?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python-output/src/result_output/mod.rs b/crates/sqlbuild-python-output/src/result_output/mod.rs new file mode 100644 index 0000000000..42aa4ebfba --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/mod.rs @@ -0,0 +1,6 @@ +//! Build result output: the execution event writer, result aggregation and documents, integration envelopes and records, resource identifiers and the terminal event index. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-output/src/result_output/models.rs b/crates/sqlbuild-python-output/src/result_output/models.rs new file mode 100644 index 0000000000..56371bb453 --- /dev/null +++ b/crates/sqlbuild-python-output/src/result_output/models.rs @@ -0,0 +1,427 @@ +//! Public records of result output. + +use crate::result_output::_helpers::events::event_writer::BUILD_OPTIONS; +use crate::result_output::_helpers::events::event_writer::within; +use crate::result_output::_helpers::events::event_writer::write; +use crate::result_output::_helpers::events::integration_records::clone_record; +use crate::result_output::_helpers::events::integration_records::degraded_record; +use crate::result_output::_helpers::events::integration_records::degraded_warning; +use crate::result_output::_helpers::events::integration_records::execution_succeeded; +use crate::result_output::_helpers::events::integration_records::integration_record; +use crate::result_output::_helpers::events::terminal_index::IndexState; +use crate::result_output::_helpers::events::terminal_index::TerminalEventClaim; +use crate::result_output::_helpers::events::terminal_index::context_variable; +use crate::result_output::_helpers::events::terminal_index::current_index_object; +use crate::result_output::_helpers::events::terminal_index::terminal_fact; +use crate::result_output::_helpers::results::projection::resource_identity; +use crate::result_output::constants::RESULT_PATH_ENV; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::pyclass; +use pyo3::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyTuple; +use sqlbuild_core::integration_results::errors::IntegrationRecordError; +use sqlbuild_core::terminal_events::main::claim_resource_terminal::claim_resource_terminal; +use sqlbuild_core::terminal_events::main::consume_terminal_fact::consume_terminal_fact; +use sqlbuild_core::terminal_events::main::latest_resource_terminal::latest_resource_terminal; +use sqlbuild_core::terminal_events::main::lifecycle_terminal::lifecycle_terminal; +use sqlbuild_core::terminal_events::models::TerminalFact; +use sqlbuild_core::terminal_events::models::TerminalIndex; +use sqlbuild_core::terminal_events::models::TerminalRequest; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::build_progress::main::progress::terminal_request::terminal_request; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::result_output::types::Checked; +use std::sync::Mutex; +use std::sync::MutexGuard; + +/// Thread-safe JSON Lines execution event writer. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct ExecutionEventWriter { + /// A `threading.Lock`; it releases the GIL while it waits. + pub(crate) lock: Py, + pub(crate) terminal_index: Option>, + /// The open text stream, until the writer is closed. + pub(crate) stream: Mutex>>, +} + +impl ExecutionEventWriter { + pub(crate) fn open_stream(&self, py: Python<'_>) -> PyResult>> { + Ok(self + .stream + .lock() + .map_err(|_| PyRuntimeError::new_err("execution event writer lock is poisoned"))? + .as_ref() + .map(|stream| stream.clone_ref(py))) + } + + /// Run `body` holding the writer lock and the index's output scope, when the stream is open. + pub(crate) fn serialized( + &self, + py: Python<'_>, + body: impl FnOnce(&Bound<'_, PyAny>) -> PyResult<()>, + ) -> PyResult<()> { + within(self.lock.bind(py), || { + let Some(stream) = self.open_stream(py)? else { + return Ok(()); + }; + match &self.terminal_index { + Some(index) => within(&index.get().output_lock(py), || body(stream.bind(py))), + None => body(stream.bind(py)), + } + }) + } + + pub(crate) fn claim( + &self, + py: Python<'_>, + request: &TerminalRequest, + ) -> PyResult> { + match &self.terminal_index { + Some(index) => index.get().claim(py, request), + None => Ok(None), + } + } + + /// Write the record, or after a contract failure warn and write the degraded record. + pub(crate) fn publish( + stream: &Bound<'_, PyAny>, + terminal: &Bound<'_, PyAny>, + record: Checked>, + degraded: impl FnOnce() -> Checked>, + ) -> PyResult<()> { + let error: IntegrationRecordError = match record? { + Ok(Some(line)) => return write(stream, &line), + Ok(None) => return Ok(()), + Err(error) => error, + }; + let stderr: Bound<'_, PyAny> = stream.py().import("sys")?.getattr("stderr")?; + stderr.call_method1( + "write", + (degraded_warning(&error, execution_succeeded(terminal)?),), + )?; + stderr.call_method0("flush")?; + match degraded()? { + Ok(Some(line)) => write(stream, &line), + Ok(None) | Err(_) => Ok(()), + } + } +} + +#[pymethods] +impl ExecutionEventWriter { + #[new] + #[pyo3(signature = (*, path = None))] + pub fn new(py: Python<'_>, path: Option>) -> PyResult { + let configured: Bound<'_, PyAny> = py + .import("os")? + .getattr("environ")? + .call_method1("get", (RESULT_PATH_ENV,))?; + let path: Option> = + match path.filter(|path| path.is_truthy().unwrap_or(true)) { + Some(path) => Some(path), + None if configured.is_truthy()? => Some( + py.import("pathlib")? + .getattr("Path")? + .call1((configured,))?, + ), + None => None, + }; + let stream: Option> = match path { + Some(path) => { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("parents", true)?; + keywords.set_item("exist_ok", true)?; + native_field(&path, "parent")?.call_method("mkdir", (), Some(&keywords))?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("mode", "a")?; + options.set_item("encoding", "utf-8")?; + Some(path.call_method("open", (), Some(&options))?.unbind()) + } + None => None, + }; + Ok(Self { + lock: py.import("threading")?.call_method0("Lock")?.unbind(), + terminal_index: current_index_object(py)?, + stream: Mutex::new(stream), + }) + } + + /// Write canonical terminal evidence plus typed build-result enrichment. + #[pyo3(signature = (**options))] + pub(crate) fn write_build_result( + &self, + py: Python<'_>, + options: Option>, + ) -> PyResult<()> { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "write_build_result", options, BUILD_OPTIONS)?; + let result: Bound<'_, PyAny> = arguments.value("result")?; + let plan: Bound<'_, PyAny> = arguments.value("plan")?; + let command: String = if arguments.passed("command")? { + arguments.value("command")?.extract()? + } else { + "build".to_owned() + }; + self.serialized(py, |stream| { + let Some((resource_name, resource_id)) = resource_identity(&result)? else { + return Ok(()); + }; + let Some(claim) = self.claim( + py, + &TerminalRequest { + resource_name, + resource_id: Some(resource_id), + ..TerminalRequest::default() + }, + )? + else { + return Ok(()); + }; + let terminal: Bound<'_, PyAny> = claim.terminal_event(py); + let sequence: usize = claim.sequence(); + Self::publish( + stream, + &terminal, + integration_record((&result, &plan), (&terminal, sequence), &command), + || degraded_record(&result, (&terminal, sequence), &command), + ) + }) + } + + /// Write canonical terminal evidence plus typed clone-result enrichment. + #[pyo3(signature = (*, item, resource_type))] + pub(crate) fn write_clone_result( + &self, + py: Python<'_>, + item: Bound<'_, PyAny>, + resource_type: &str, + ) -> PyResult<()> { + self.serialized(py, |stream| { + let name: String = native_field(&item, "name")?.extract()?; + let Some(claim) = self.claim( + py, + &TerminalRequest { + resource_id: Some(format!("{resource_type}:{name}")), + resource_name: name, + ..TerminalRequest::default() + }, + )? + else { + return Ok(()); + }; + let terminal: Bound<'_, PyAny> = claim.terminal_event(py); + let sequence: usize = claim.sequence(); + Self::publish( + stream, + &terminal, + Ok(clone_record(&item, (&terminal, sequence))?.map(Some)), + || degraded_record(&item, (&terminal, sequence), "clone"), + ) + }) + } + + /// Close the event stream when configured. + pub(crate) fn close(&self, py: Python<'_>) -> PyResult<()> { + within(self.lock.bind(py), || { + let Some(stream) = self.open_stream(py)? else { + return Ok(()); + }; + let close = || -> PyResult<()> { + *self.stream.lock().map_err(|_| { + PyRuntimeError::new_err("execution event writer lock is poisoned") + })? = None; + stream.bind(py).call_method0("close")?; + Ok(()) + }; + match &self.terminal_index { + Some(index) => within(&index.get().output_lock(py), close), + None => close(), + } + }) + } +} + +/// Retain canonical facts and claim resource terminals for result projection. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct TerminalEventIndex { + pub(crate) state: Mutex, + pub(crate) output_lock: Py, +} + +impl TerminalEventIndex { + pub(crate) fn locked(&self) -> PyResult> { + self.state + .lock() + .map_err(|_| PyRuntimeError::new_err("terminal event index lock is poisoned")) + } + + pub(crate) fn event_at( + &self, + py: Python<'_>, + sequence: Option, + ) -> PyResult>> { + let state: MutexGuard<'_, IndexState> = self.locked()?; + Ok(sequence.map(|sequence| state.events[sequence].clone_ref(py))) + } + + /// The latest terminal of a resource without claiming it. + pub fn latest_terminal<'py>( + &self, + py: Python<'py>, + resource_name: &str, + resource_id: Option<&str>, + ) -> PyResult>> { + let sequence: Option = { + let state: MutexGuard<'_, IndexState> = self.locked()?; + latest_resource_terminal(&state.index, resource_name, resource_id) + }; + Ok(self + .event_at(py, sequence)? + .map(|event| event.into_bound(py))) + } + + /// The strongest aggregate terminal for the final result document. + pub fn aggregate_terminal<'py>(&self, py: Python<'py>) -> PyResult>> { + let sequence: Option = lifecycle_terminal(&self.locked()?.index); + Ok(self + .event_at(py, sequence)? + .map(|event| event.into_bound(py))) + } + + /// Claim the terminal a result callback reports on. + pub(crate) fn claim( + &self, + py: Python<'_>, + request: &TerminalRequest, + ) -> PyResult> { + let mut state: MutexGuard<'_, IndexState> = self.locked()?; + let (index, claimed) = claim_resource_terminal(std::mem::take(&mut state.index), request); + state.index = index; + let Some(sequence) = claimed else { + return Ok(None); + }; + Ok(Some(TerminalEventClaim { + terminal: state.events[sequence].clone_ref(py), + event_sequence: sequence, + })) + } + + /// The reentrant lock that serializes claims with physical result output. + pub fn output_lock<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { + self.output_lock.bind(py).clone() + } +} + +#[pymethods] +impl TerminalEventIndex { + #[new] + pub fn new(py: Python<'_>) -> PyResult { + Ok(Self { + state: Mutex::new(IndexState { + index: TerminalIndex::default(), + events: Vec::new(), + }), + output_lock: py.import("threading")?.call_method0("RLock")?.unbind(), + }) + } + + /// Store a known fact once in dispatcher publication order. + pub(crate) fn consume(&self, event: &Bound<'_, PyAny>) -> PyResult<()> { + let fact: TerminalFact = terminal_fact(event)?; + let mut state: MutexGuard<'_, IndexState> = self.locked()?; + let (index, stored) = consume_terminal_fact(std::mem::take(&mut state.index), fact); + state.index = index; + if stored.is_some() { + state.events.push(event.clone().unbind()); + } + Ok(()) + } + + /// Claim the latest exact terminal available when a result callback arrives. + #[pyo3(signature = (**options))] + pub(crate) fn claim_resource_terminal( + &self, + py: Python<'_>, + options: Option>, + ) -> PyResult> { + self.claim( + py, + &terminal_request(py, "claim_resource_terminal", options)?, + ) + } + + /// Return the latest matching terminal without claiming it. + #[pyo3(signature = (*, resource_name, resource_id = None))] + pub(crate) fn resource_terminal<'py>( + &self, + py: Python<'py>, + resource_name: &str, + resource_id: Option<&str>, + ) -> PyResult>> { + self.latest_terminal(py, resource_name, resource_id) + } + + /// Return the strongest aggregate terminal available to final JSON. + pub(crate) fn lifecycle_terminal<'py>( + &self, + py: Python<'py>, + ) -> PyResult>> { + self.aggregate_terminal(py) + } + + /// Serialize terminal claims with physical integration-result output. + pub(crate) fn output_serialization_scope<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { + self.output_lock(py) + } + + /// Return idempotent canonical facts in publication order. + pub(crate) fn events<'py>(&self, py: Python<'py>) -> PyResult> { + let state: MutexGuard<'_, IndexState> = self.locked()?; + PyTuple::new(py, state.events.iter().map(|event| event.bind(py))) + } +} + +/// Installs one index for the duration of a `with` block. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct TerminalEventIndexScope { + pub(crate) index: Py, + pub(crate) token: Mutex>>, +} + +#[pymethods] +impl TerminalEventIndexScope { + pub(crate) fn __enter__(&self, py: Python<'_>) -> PyResult> { + let token: Bound<'_, PyAny> = context_variable(py)?.call_method1("set", (&self.index,))?; + *self.token.lock().map_err(|_| { + PyRuntimeError::new_err("terminal event index scope lock is poisoned") + })? = Some(token.unbind()); + Ok(self.index.clone_ref(py)) + } + + #[pyo3(signature = (*_exc_info))] + pub(crate) fn __exit__( + &self, + py: Python<'_>, + _exc_info: &Bound<'_, PyTuple>, + ) -> PyResult { + let token: Option> = self + .token + .lock() + .map_err(|_| PyRuntimeError::new_err("terminal event index scope lock is poisoned"))? + .take(); + if let Some(token) = token { + context_variable(py)?.call_method1("reset", (token,))?; + } + Ok(false) + } +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index a9acd24ca8..123320ddd2 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -39,6 +39,7 @@ sqlbuild-python-front.workspace = true sqlbuild-python-project.workspace = true sqlbuild-python-planner.workspace = true sqlbuild-python-compile.workspace = true +sqlbuild-python-output.workspace = true sqlbuild-recall.workspace = true sqlbuild-refactor.workspace = true sqlbuild-rules.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 09106652cc..49686bc982 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -16,18 +16,9 @@ use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_host::models::{AuditExecutionResult, AuditOutcome}; use sqlbuild_orchestration::runs::models::NodeOutcome; -use crate::bindings::_helpers::build_progress::audit_progress::AuditProgressReporter; -use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; -use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::result_output::documents::{ - audit_execution_json, audit_projection, write_execution_json_output, -}; -use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; -use crate::bindings::_helpers::result_output::projection::audit_check_id; -use crate::bindings::_helpers::result_output::resource_ids::audit_identity; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::main::within::within; @@ -41,6 +32,15 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::build_static_pipeline_result::build_static_pipeline_result; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; use sqlbuild_python_front::cost_context::models::CostContext; +use sqlbuild_python_output::build_progress::main::write_execution_header::write_execution_header; +use sqlbuild_python_output::build_progress::models::AuditProgressReporter; +use sqlbuild_python_output::build_progress::models::ConnectionProgressReporter; +use sqlbuild_python_output::result_output::main::events::audit_identity::audit_identity; +use sqlbuild_python_output::result_output::main::results::audit_check_id::audit_check_id; +use sqlbuild_python_output::result_output::main::results::audit_execution_json::audit_execution_json; +use sqlbuild_python_output::result_output::main::results::audit_projection::audit_projection; +use sqlbuild_python_output::result_output::main::results::write_execution_json_output::write_execution_json_output; +use sqlbuild_python_output::result_output::models::ExecutionEventWriter; use sqlbuild_python_planner::planner_entries::main::plans::build_audit_command_plan::build_audit_command_plan; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; use sqlbuild_python_project::project_snapshot::types::VerifiedRun; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index f23f49ab87..866799cfe4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -19,7 +19,6 @@ use crate::bindings::_helpers::build_command::planning::{ write_compile_target, write_plan_text, }; use crate::bindings::_helpers::check::check_core::relevant_check_functions; -use crate::bindings::_helpers::recall::console::write_stderr; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::build_command::main::frozenset::frozenset; @@ -34,6 +33,7 @@ use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_use use sqlbuild_python_front::discovery::main::sources::provider_uses::provider_uses; use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; +use sqlbuild_python_output::recall::main::write_stderr::write_stderr; use sqlbuild_python_planner::observations::main::arm_source_capture::arm_source_capture; use sqlbuild_python_planner::observations::main::take_source_capture::take_source_capture; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index c16a49fd10..9d3546c385 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -16,16 +16,11 @@ use crate::bindings::_helpers::build_command::execution::{ Outcome, Preparation, write_run_context, }; use crate::bindings::_helpers::build_command::invocation::BuildContext; -use crate::bindings::_helpers::build_progress::footer::format_build_footer_with; -use crate::bindings::_helpers::build_progress::result_text::build_result_text; use crate::bindings::_helpers::check::check_results::check_result_text; use crate::bindings::_helpers::cost_report::degradation::{ finalize_build_cost as finalize_cost, render_build_cost, }; use crate::bindings::_helpers::cost_report::functions::cost_output_payload; -use crate::bindings::_helpers::result_output::documents::format_build_execution_json; -use crate::bindings::_helpers::result_output::documents::format_build_execution_json_with; -use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use sqlbuild_core::build_progress::models::PythonNodeText; use sqlbuild_core::check_command::models::{CheckResultText, CheckStatus}; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -43,6 +38,11 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::plan_has_executable_work::plan_has_executable_work; use sqlbuild_python_compile::compile_outputs::main::write_python_check_runtime_target::write_python_check_runtime_target; use sqlbuild_python_compile::compile_outputs::main::write_runtime_target::write_runtime_target; +use sqlbuild_python_output::build_progress::main::build_result_text::build_result_text; +use sqlbuild_python_output::build_progress::main::format_build_footer_with::format_build_footer_with; +use sqlbuild_python_output::result_output::main::results::format_build_execution_json::format_build_execution_json; +use sqlbuild_python_output::result_output::main::results::format_build_execution_json_with::format_build_execution_json_with; +use sqlbuild_python_output::result_output::main::results::write_execution_json_output::write_execution_json_output; const SUCCESS_STATUS: &str = "success"; const CHECK_KIND: &str = "check"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 82138f625b..c4bebd360d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -15,11 +15,6 @@ use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_command::lifecycle::{ DirectPythonLifecycle, LifecycleInputs, prepare_direct_python_lifecycle, }; -use crate::bindings::_helpers::build_progress::build_callbacks::{ - BuildProgressCallbacks, batch_complete_callback, check_callback, -}; -use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; -use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::check::check_core::{ build_check_relation_targets, load_results_by_loader_name, record_python_run_state_results, relevant_check_functions, @@ -53,6 +48,11 @@ use sqlbuild_python_front::cost_context::models::CostContext; use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; use sqlbuild_python_front::native_host::main::recorder_callbacks::recorder_callbacks; +use sqlbuild_python_output::build_progress::main::batch_complete_callback::batch_complete_callback; +use sqlbuild_python_output::build_progress::main::check_callback::check_callback; +use sqlbuild_python_output::build_progress::main::write_execution_header::write_execution_header; +use sqlbuild_python_output::build_progress::models::BuildProgressCallbacks; +use sqlbuild_python_output::build_progress::models::ConnectionProgressReporter; use sqlbuild_python_planner::consumption::main::changed_policy::changed_policy; use sqlbuild_python_planner::consumption::main::held_back_text::held_back_text; use sqlbuild_python_planner::consumption::main::policy_line::policy_line; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 68adf86f23..b67d47b3b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -10,8 +10,6 @@ use sqlbuild_core::build_command::main::duration_limit_error::duration_limit_err use sqlbuild_core::build_command::models::DurationLimit; use crate::bindings::_helpers::build_command::policies::build_command_error; -use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; -use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -31,6 +29,8 @@ use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::reso use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; use sqlbuild_python_front::discovery::models::SecretPolicy; +use sqlbuild_python_output::build_progress::models::ConnectionProgressReporter; +use sqlbuild_python_output::build_progress::models::PlanningProgressReporter; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; const DEFER_CONFLICT_CODE: &str = "C408"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index 2ff106622a..e843ca0896 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -11,7 +11,6 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::build_command::defer_clone::run_defer_clone_boundary_prephase; use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_command::policies::{enforce_plan_policies, model_limit}; -use crate::bindings::_helpers::build_progress::progress_text::with_verbose_inspection; use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -26,6 +25,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::manifest::main::build_manifest::build_manifest; +use sqlbuild_python_output::build_progress::main::with_verbose_inspection::with_verbose_inspection; use sqlbuild_python_planner::plan_output::main::format_plan::format_plan; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs deleted file mode 100644 index 4494babe1b..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs +++ /dev/null @@ -1,313 +0,0 @@ -//! Aggregate live state and plan-ordered result rows of standalone concurrent audits. - -use std::collections::HashMap; -use std::sync::{Mutex, MutexGuard, PoisonError}; - -use pyo3::exceptions::PyKeyError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyDict; -use pyo3::{pyclass, pymethods}; -use sqlbuild_core::build_progress::models::AuditResultText; -use sqlbuild_core::cli_text::models::CliStyle; -use sqlbuild_core::nested_progress::main::audit_group_name::audit_group_name; -use sqlbuild_core::nested_progress::main::audit_resource_id::audit_resource_id; -use sqlbuild_core::nested_progress::main::render_audit_aggregate::render_audit_aggregate; -use sqlbuild_core::nested_progress::main::render_audit_row::render_audit_row; -use sqlbuild_core::nested_progress::main::render_group_header::render_group_header; -use sqlbuild_core::nested_progress::models::AuditAggregate; - -use crate::bindings::_helpers::build_progress::result_text::{audit_text, optional, text}; -use sqlbuild_python_base::boundary::main::within::within; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::build_progress::constants::CLEAR_LINE; -use sqlbuild_python_base::build_progress::constants::HIDE_CURSOR; -use sqlbuild_python_base::build_progress::constants::SHOW_CURSOR; -use sqlbuild_python_base::build_progress::main::terminal::claim::claim; -use sqlbuild_python_base::build_progress::main::terminal::consume_terminal::consume_terminal; -use sqlbuild_python_base::build_progress::main::terminal::flush::flush; -use sqlbuild_python_base::build_progress::main::terminal::projector::projector; -use sqlbuild_python_base::build_progress::main::terminal::release::release; -use sqlbuild_python_base::build_progress::main::terminal::shared_lines::shared_lines; -use sqlbuild_python_base::build_progress::main::terminal::tty_flag::tty_flag; -use sqlbuild_python_base::build_progress::main::terminal::write::write; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -/// The identity an audit plan entry or result reports to the progress projector. -fn resource_id(value: &Bound<'_, PyAny>, name_field: &str) -> PyResult { - let kind: Option = match optional(value, "attached_target_kind")? { - Some(kind) => Some(kind), - None => Some(text(value, "attachment_kind")?), - }; - Ok(audit_resource_id( - &text(value, name_field)?, - kind.as_deref().unwrap_or(""), - ( - optional(value, "attached_target_name")?.as_deref(), - optional(value, "attached_column_name")?.as_deref(), - ), - )) -} - -/// Mutable reporter state; only held briefly and never across Python calls. -#[derive(Default)] -struct ReporterState { - completed_by_index: HashMap>>, - next_index: usize, - running: usize, - completed: usize, - current_group: Option, - closed: bool, - cursor_hidden: bool, - on_result: Option>, -} - -/// Render aggregate live state and flush completed audit rows in plan order. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct AuditProgressReporter { - stream: Py, - lines: Py, - lock: Py, - style: CliStyle, - is_tty: bool, - projector: Option>, - indexes: HashMap, - total: usize, - worker_limit: i64, - state: Mutex, -} - -impl AuditProgressReporter { - fn state(&self) -> MutexGuard<'_, ReporterState> { - self.state.lock().unwrap_or_else(PoisonError::into_inner) - } - - fn index_of(&self, identity: String) -> PyResult { - self.indexes - .get(&identity) - .copied() - .ok_or_else(|| PyKeyError::new_err(identity)) - } - - fn render_aggregate(slf: &Bound<'_, Self>) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let reporter: &Self = slf.get(); - let (closed, cursor_hidden, running, completed) = { - let state = reporter.state(); - ( - state.closed, - state.cursor_hidden, - state.running, - state.completed, - ) - }; - if !reporter.is_tty || closed { - return Ok(()); - } - let stream: &Bound<'_, PyAny> = reporter.stream.bind(py); - claim(reporter.lines.bind(py), stream, slf.as_any())?; - if !cursor_hidden { - write(stream, HIDE_CURSOR)?; - reporter.state().cursor_hidden = true; - } - write( - stream, - &render_audit_aggregate(AuditAggregate { - running, - completed, - total: reporter.total, - workers: reporter.worker_limit, - }), - )?; - flush(stream) - } - - fn flush_ready(&self, py: Python<'_>) -> PyResult<()> { - loop { - let ready: Option> = { - let mut state = self.state(); - let index: usize = state.next_index; - let Some(result) = state.completed_by_index.remove(&index) else { - return Ok(()); - }; - state.next_index += 1; - result - }; - if let Some(result) = ready { - self.write_result(py, result.bind(py))?; - } - } - } - - fn write_result(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { - let stream: &Bound<'_, PyAny> = self.stream.bind(py); - if self.is_tty { - write(stream, CLEAR_LINE)?; - } - let audit: AuditResultText = audit_text(result)?; - let group: String = audit_group_name(&audit); - let previous: Option = self.state().current_group.clone(); - if previous.as_ref() != Some(&group) { - write( - stream, - &render_group_header(&group, previous.is_none(), self.style), - )?; - self.state().current_group = Some(group); - } - let duration_ms: Option = match &self.projector { - Some(projector) => consume_terminal( - projector.bind(py), - &audit.audit_name, - Some(&resource_id(result, "audit_name")?), - )?, - None => None, - }; - write(stream, &render_audit_row(&audit, duration_ms, self.style))?; - flush(stream)?; - let callback: Option> = self - .state() - .on_result - .as_ref() - .map(|callback| callback.clone_ref(py)); - if let Some(callback) = callback { - callback.bind(py).call1((result,))?; - } - Ok(()) - } - - fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { - within(self.lock.bind(py), |_| body()) - } -} - -#[pymethods] -impl AuditProgressReporter { - #[new] - #[pyo3(signature = (**options))] - pub(crate) fn new(py: Python<'_>, options: Option>) -> PyResult { - let arguments = KeywordArguments::read( - py, - "AuditProgressReporter", - options, - (&["entries", "worker_limit", "stream", "use_color"], &[]), - )?; - let stream: Bound<'_, PyAny> = arguments.value("stream")?; - let lines: Bound<'_, PyAny> = shared_lines(py)?; - let mut indexes: HashMap = HashMap::new(); - let mut total: usize = 0; - for entry in arguments.value("entries")?.try_iter()? { - indexes.insert(resource_id(&entry?, "name")?, total); - total += 1; - } - Ok(Self { - is_tty: tty_flag(&stream)?, - stream: stream.unbind(), - lock: native_field(&lines, "lock")?.unbind(), - lines: lines.unbind(), - style: CliStyle { - use_color: arguments.value("use_color")?.is_truthy()?, - }, - projector: projector(py)?, - indexes, - total, - worker_limit: arguments.value("worker_limit")?.extract()?, - state: Mutex::new(ReporterState::default()), - }) - } - - /// Install plan-ordered result enrichment before dispatch starts. - fn set_result_callback(&self, callback: Py) { - self.state().on_result = Some(callback); - } - - /// Record a worker start without mutating one-item spinner state. - fn on_item_start(slf: &Bound<'_, Self>, entry: &Bound<'_, PyAny>) -> PyResult<()> { - let _ = entry; - slf.get().locked(slf.py(), || { - slf.get().state().running += 1; - Self::render_aggregate(slf) - }) - } - - /// Flush every newly contiguous plan-order result row. - fn on_item_complete(slf: &Bound<'_, Self>, result: &Bound<'_, PyAny>) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let reporter: &Self = slf.get(); - reporter.locked(py, || { - let index: usize = reporter.index_of(resource_id(result, "audit_name")?)?; - reporter - .state() - .completed_by_index - .insert(index, Some(result.clone().unbind())); - reporter.flush_ready(py)?; - Self::render_aggregate(slf) - }) - } - - /// Update aggregate state immediately without projecting a result row. - fn on_item_physical_complete(slf: &Bound<'_, Self>, result: &Bound<'_, PyAny>) -> PyResult<()> { - let _ = result; - slf.get().locked(slf.py(), || { - { - let mut state = slf.get().state(); - state.running = state.running.saturating_sub(1); - state.completed += 1; - } - Self::render_aggregate(slf) - }) - } - - /// Advance aggregate and ordering state for a fatal audit exception. - fn on_item_error(slf: &Bound<'_, Self>, entry: &Bound<'_, PyAny>) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let reporter: &Self = slf.get(); - reporter.locked(py, || { - let index: usize = reporter.index_of(resource_id(entry, "name")?)?; - { - let mut state = reporter.state(); - state.running = state.running.saturating_sub(1); - state.completed += 1; - state.completed_by_index.insert(index, None); - } - reporter.flush_ready(py)?; - Self::render_aggregate(slf) - }) - } - - /// Restore the terminal after the final aggregate update. - fn close(slf: &Bound<'_, Self>) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let reporter: &Self = slf.get(); - reporter.locked(py, || { - reporter.state().closed = true; - release(reporter.lines.bind(py), slf.as_any())?; - if !reporter.is_tty { - return Ok(()); - } - let stream: &Bound<'_, PyAny> = reporter.stream.bind(py); - write(stream, CLEAR_LINE)?; - let hidden: bool = std::mem::take(&mut reporter.state().cursor_hidden); - if hidden { - write(stream, SHOW_CURSOR)?; - } - flush(stream) - }) - } - - /// Erase the aggregate row so a persistent line can take its place. - fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { - self.locked(py, || { - let stream: &Bound<'_, PyAny> = self.stream.bind(py); - write(stream, CLEAR_LINE)?; - flush(stream) - }) - } - - /// Draw the aggregate row again below persistent output. - fn redraw_transient_line(slf: &Bound<'_, Self>) -> PyResult<()> { - slf.get().locked(slf.py(), || Self::render_aggregate(slf)) - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs deleted file mode 100644 index 2ca936d586..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs +++ /dev/null @@ -1,722 +0,0 @@ -//! Live progress rows, completion rows and runtime diagnostics of `sqb build`. - -use std::collections::HashMap; -use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; -use std::time::Instant; - -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyTuple, PyTupleMethods}; -use pyo3::{pyclass, pymethods}; -use sqlbuild_core::build_progress::main::batch_progress_message::batch_progress_message; -use sqlbuild_core::build_progress::main::query_diagnostic_message::query_diagnostic_message; -use sqlbuild_core::build_progress::main::render_build_spinner_line::render_build_spinner_line; -use sqlbuild_core::build_progress::main::render_failed_test_result::render_failed_test_result; -use sqlbuild_core::build_progress::main::render_model_result::render_model_result; -use sqlbuild_core::build_progress::main::render_resource_result::render_resource_result; -use sqlbuild_core::build_progress::main::runtime_diagnostic_summary::runtime_diagnostic_summary; -use sqlbuild_core::build_progress::main::scheduler_diagnostic_message::scheduler_diagnostic_message; -use sqlbuild_core::build_progress::models::{ - BatchProgressText, ModelRowFacts, ProgressLayout, ResourceResult, SchedulerStateText, - SpinnerRow, StatementText, TestResultText, -}; -use sqlbuild_core::cli_text::models::CliStyle; - -use crate::bindings::_helpers::build_progress::plan_facts::{BuildPlanFacts, build_plan_facts}; -use crate::bindings::_helpers::build_progress::result_text::{ - function_text, load_text, model_text, optional, seed_text, test_text, text, -}; -use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; -use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::main::within::within; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::build_progress::constants::CLEAR_LINE; -use sqlbuild_python_base::build_progress::constants::HIDE_CURSOR; -use sqlbuild_python_base::build_progress::constants::SHOW_CURSOR; -use sqlbuild_python_base::build_progress::constants::SPINNER_FRAMES; -use sqlbuild_python_base::build_progress::constants::SPINNER_TICK_SECONDS; -use sqlbuild_python_base::build_progress::main::terminal::claim::claim; -use sqlbuild_python_base::build_progress::main::terminal::consume_terminal::consume_terminal; -use sqlbuild_python_base::build_progress::main::terminal::flush::flush; -use sqlbuild_python_base::build_progress::main::terminal::join_spinner::join_spinner; -use sqlbuild_python_base::build_progress::main::terminal::projector::projector; -use sqlbuild_python_base::build_progress::main::terminal::release::release; -use sqlbuild_python_base::build_progress::main::terminal::shared_lines::shared_lines; -use sqlbuild_python_base::build_progress::main::terminal::start_spinner::start_spinner; -use sqlbuild_python_base::build_progress::main::terminal::terminal_columns::terminal_columns; -use sqlbuild_python_base::build_progress::main::terminal::tty_flag::tty_flag; -use sqlbuild_python_base::build_progress::main::terminal::write::write; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_keywords; -use sqlbuild_python_project::project_snapshot::models::WorkerBuild; - -const RUNTIME_LOGGER: &str = "sqlbuild.executor.runtime"; -const PASS_OUTCOME: &str = "pass"; -const SCHEDULER_DIAGNOSTIC_LIMIT: usize = 50; -const QUERY_DIAGNOSTIC_LIMIT: usize = 25; - -/// Which executor result type a completion reports. -#[derive(Clone, Copy, PartialEq, Eq)] -enum ResultKind { - Model, - Seed, - Function, - Load, - PythonNode, - SqlTest, - Other, -} - -fn result_kind(result: &Bound<'_, PyAny>) -> PyResult { - for (field, kind) in [ - ("model_name", ResultKind::Model), - ("seed_name", ResultKind::Seed), - ("function_name", ResultKind::Function), - ("loader_name", ResultKind::Load), - ("node_name", ResultKind::PythonNode), - ("test_name", ResultKind::SqlTest), - ] { - if result.hasattr(field)? { - return Ok(kind); - } - } - Ok(ResultKind::Other) -} - -fn sql_test_resource_id(result: &Bound<'_, PyAny>) -> PyResult { - let name: String = text(result, "test_name")?; - let path: Bound<'_, PyAny> = native_field(result, "source_path")?; - let block: Bound<'_, PyAny> = native_field(result, "block_index")?; - let case: Bound<'_, PyAny> = native_field(result, "case_name")?; - if path.is_none() || block.is_none() || case.is_none() { - return Ok(format!("sql_test:{name}")); - } - Ok(format!( - "sql_test:{}:{}:{}", - path.call_method0("as_posix")?.str()?, - block.str()?, - case.str()? - )) -} - -/// The resource name and projector identity of a completion. -pub(crate) fn resource_identity(result: &Bound<'_, PyAny>) -> PyResult> { - Ok(match result_kind(result)? { - ResultKind::Model => { - let name: String = text(result, "model_name")?; - Some((name.clone(), format!("model:{name}"))) - } - ResultKind::Seed => { - let name: String = text(result, "seed_name")?; - Some((name.clone(), format!("seed:{name}"))) - } - ResultKind::Function => { - let name: String = text(result, "function_name")?; - let kind: String = text(result, "function_kind")?; - Some((name.clone(), format!("{kind}:{name}"))) - } - ResultKind::Load => { - let name: String = text(result, "source_name")?; - Some((name.clone(), format!("source:{name}"))) - } - ResultKind::PythonNode => { - let name: String = text(result, "node_name")?; - let kind: String = text(result, "kind")?; - Some((name.clone(), format!("{kind}:{name}"))) - } - ResultKind::SqlTest => Some((text(result, "test_name")?, sql_test_resource_id(result)?)), - ResultKind::Other => None, - }) -} - -/// Mutable callback state; only held briefly and never across Python calls. -#[derive(Default)] -struct CallbackState { - counter: usize, - node_name: String, - node_kind: String, - sub_message: String, - frame: usize, - line_active: bool, - cursor_hidden: bool, - stop_event: Option>, - thread: Option>, - tests_by_model: HashMap>, - scheduler_count: usize, - scheduler_omitted: usize, - last_scheduler: Option, - query_counts: HashMap, - query_omitted: usize, - last_query: Option, -} - -/// Encapsulates live build progress output state and callbacks. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct BuildProgressCallbacks { - pub(crate) worker: Option>, - plan: Py, - stream: Py, - lines: Py, - lock: Py, - event_writer: Py, - projector: Option>, - facts: BuildPlanFacts, - style: CliStyle, - verbose: bool, - debug: bool, - is_tty: bool, - started: Instant, - state: Mutex, -} - -impl BuildProgressCallbacks { - fn state(&self) -> MutexGuard<'_, CallbackState> { - self.state.lock().unwrap_or_else(PoisonError::into_inner) - } - - fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { - within(self.lock.bind(py), |_| body()) - } - - fn layout(&self, py: Python<'_>) -> PyResult { - Ok(ProgressLayout { - total: self.facts.total, - name_width: self.facts.name_width, - columns: terminal_columns(py)?, - verbose: self.verbose, - }) - } - - fn emit(&self, py: Python<'_>, text: &str) -> PyResult<()> { - let stream: &Bound<'_, PyAny> = self.stream.bind(py); - write(stream, text)?; - flush(stream) - } - - fn write_spinner_line(&self, py: Python<'_>) -> PyResult<()> { - self.locked(py, || { - let (row, frame) = { - let mut state = self.state(); - if !state.line_active { - return Ok(()); - } - let frame: usize = state.frame; - state.frame = (frame + 1) % SPINNER_FRAMES.len(); - ( - SpinnerRow { - position: state.counter + 1, - resource_kind: state.node_kind.clone(), - name: state.node_name.clone(), - message: state.sub_message.clone(), - }, - frame, - ) - }; - self.emit( - py, - &render_build_spinner_line(&row, frame, self.layout(py)?, self.style), - ) - }) - } - - fn set_cursor(&self, py: Python<'_>, hidden: bool) -> PyResult<()> { - if self.state().cursor_hidden == hidden { - return Ok(()); - } - self.locked(py, || { - self.emit(py, if hidden { HIDE_CURSOR } else { SHOW_CURSOR }) - })?; - self.state().cursor_hidden = hidden; - Ok(()) - } - - fn stop_spinner(&self, py: Python<'_>) -> PyResult<()> { - let (event, thread) = { - let mut state = self.state(); - (state.stop_event.take(), state.thread.take()) - }; - if let Some(event) = event { - event.bind(py).call_method0("set")?; - } - if let Some(thread) = thread { - join_spinner(thread.bind(py))?; - } - Ok(()) - } - - /// Stop the spinner and erase its row before a persistent row is written. - fn retire_live_row(slf: &Bound<'_, Self>) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let callbacks: &Self = slf.get(); - callbacks.stop_spinner(py)?; - if !callbacks.is_tty { - return Ok(()); - } - callbacks.locked(py, || { - callbacks.state().line_active = false; - release(callbacks.lines.bind(py), slf.as_any())?; - callbacks.emit(py, CLEAR_LINE) - })?; - callbacks.set_cursor(py, false) - } - - fn write_runtime_diagnostic(&self, py: Python<'_>, message: &str) -> PyResult<()> { - imported(py, "logging", "getLogger")? - .call1((RUNTIME_LOGGER,))? - .call_method1("debug", (message,))?; - if self.debug { - return Ok(()); - } - self.write_persistent(py, message) - } - - /// Write one permanent line through the shared line coordinator. - fn write_persistent(&self, py: Python<'_>, message: &str) -> PyResult<()> { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("stream", self.stream.bind(py))?; - options.set_item("text", format!("{message}\n"))?; - self.lines - .bind(py) - .call_method("write_persistent", (), Some(&options))?; - Ok(()) - } - - /// Keep a line per completed microbatch when no live spinner row shows batch progress. - fn write_batch_complete(&self, py: Python<'_>, progress: &Bound<'_, PyAny>) -> PyResult<()> { - if self.is_tty { - return Ok(()); - } - let batch: BatchProgressText = BatchProgressText { - model_name: text(progress, "model_name")?, - batch_number: native_field(progress, "batch_number")?.extract()?, - batch_count: native_field(progress, "batch_count")?.extract()?, - window: text(progress, "window")?, - rows_affected: native_field(progress, "rows_affected")?.extract()?, - elapsed_seconds: native_field(progress, "elapsed_seconds")?.extract()?, - }; - self.write_persistent(py, &batch_progress_message(&batch)) - } - - fn write_node_result( - slf: &Bound<'_, Self>, - result: &Bound<'_, PyAny>, - kind: ResultKind, - ) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let callbacks: &Self = slf.get(); - if kind == ResultKind::SqlTest { - let test: TestResultText = test_text(result)?; - if test.outcome != PASS_OUTCOME { - Self::retire_live_row(slf)?; - return callbacks.emit( - py, - &render_failed_test_result(&test, callbacks.layout(py)?, callbacks.style), - ); - } - let mut state = callbacks.state(); - for step in &test.steps { - state - .tests_by_model - .entry(step.model_name.clone()) - .or_default() - .push(test.clone()); - } - return Ok(()); - } - Self::retire_live_row(slf)?; - let counter: usize = { - let mut state = callbacks.state(); - state.counter += 1; - state.counter - }; - let layout: ProgressLayout = callbacks.layout(py)?; - let style: CliStyle = callbacks.style; - let text: String = match kind { - ResultKind::Seed => render_resource_result( - ResourceResult::Node(&seed_text(result)?), - counter, - layout, - style, - ), - ResultKind::Function => render_resource_result( - ResourceResult::Node(&function_text(result)?), - counter, - layout, - style, - ), - ResultKind::Load => render_resource_result( - ResourceResult::Load(&load_text(result)?), - counter, - layout, - style, - ), - ResultKind::Model => { - let model = model_text(result)?; - let facts: &ModelRowFacts = callbacks - .facts - .models - .get(&model.name) - .unwrap_or(&callbacks.facts.unplanned_model); - let tests: Vec = callbacks - .state() - .tests_by_model - .get(&model.name) - .cloned() - .unwrap_or_default(); - render_model_result((&model, facts, &tests), counter, layout, style) - } - _ => return Ok(()), - }; - callbacks.emit(py, &text) - } - - fn write_event(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { - let writer: &Bound<'_, PyAny> = self.event_writer.bind(py); - let write_one = |item: &Bound<'_, PyAny>| -> PyResult<()> { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("result", item)?; - options.set_item("plan", self.plan.bind(py))?; - writer.call_method("write_build_result", (), Some(&options))?; - Ok(()) - }; - write_one(result)?; - if result_kind(result)? == ResultKind::Model { - for audit in native_field(result, "audit_results")?.try_iter()? { - write_one(&audit?)?; - } - } - Ok(()) - } -} - -fn scheduler_state(state: &Bound<'_, PyAny>) -> PyResult { - Ok(SchedulerStateText { - running: native_field(state, "running")?.extract()?, - ready: native_field(state, "ready")?.extract()?, - waiting: native_field(state, "waiting")?.extract()?, - limit: native_field(state, "limit")?.extract()?, - aborted: native_field(state, "aborted")?.extract()?, - }) -} - -fn statement(telemetry: &Bound<'_, PyAny>) -> PyResult { - Ok(StatementText { - query_id: optional(telemetry, "query_id")?, - status: text(telemetry, "status")?, - elapsed_seconds: native_field(telemetry, "elapsed_seconds")?.extract()?, - resource_type: text(telemetry, "resource_type")?, - resource_name: text(telemetry, "resource_name")?, - phase: text(telemetry, "phase")?, - }) -} - -#[pymethods] -impl BuildProgressCallbacks { - #[new] - #[pyo3(signature = (**options))] - pub(crate) fn new(py: Python<'_>, options: Option>) -> PyResult { - let arguments = KeywordArguments::read( - py, - "BuildProgressCallbacks", - options, - ( - &["plan", "use_color"], - &["verbose", "debug", "event_output_path"], - ), - )?; - let flag = |name: &str| -> PyResult { - Ok(arguments.passed(name)? && arguments.value(name)?.is_truthy()?) - }; - let plan: Bound<'_, PyAny> = arguments.value("plan")?; - let debug: bool = flag("debug")?; - let sys: Bound<'_, PyAny> = py.import("sys")?.into_any(); - let stdout: Bound<'_, PyAny> = native_field(&sys, "stdout")?; - let stream: Bound<'_, PyAny> = if debug { - native_field(&sys, "stderr")? - } else { - stdout.clone() - }; - let path: Bound<'_, PyAny> = if arguments.passed("event_output_path")? { - arguments.value("event_output_path")? - } else { - py.None().into_bound(py) - }; - let event_writer: Bound<'_, PyAny> = Bound::new( - py, - ExecutionEventWriter::new(py, Some(path).filter(|value| !value.is_none()))?, - )? - .into_any(); - let facts: BuildPlanFacts = build_plan_facts(&plan)?; - let projector: Option> = projector(py)?; - if let Some(projector) = &projector { - let projector: &Bound<'_, PyAny> = projector.bind(py); - let ordinals: Bound<'_, PyDict> = PyDict::new(py); - for (name, ordinal) in &facts.ordinals { - ordinals.set_item(name, ordinal)?; - } - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("ordinals", ordinals)?; - options.set_item("total", facts.total)?; - projector.call_method("configure_resources", (), Some(&options))?; - for name in &facts.test_names { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("resource_name", name)?; - projector.call_method("expect_resource_enrichment", (), Some(&options))?; - } - } - let lines: Bound<'_, PyAny> = shared_lines(py)?; - Ok(Self { - worker: None, - plan: plan.unbind(), - is_tty: tty_flag(&stdout)? && !debug, - stream: stream.unbind(), - lock: native_field(&lines, "lock")?.unbind(), - lines: lines.unbind(), - event_writer: event_writer.unbind(), - projector, - facts, - style: CliStyle { - use_color: arguments.value("use_color")?.is_truthy()?, - }, - verbose: flag("verbose")?, - debug, - started: Instant::now(), - state: Mutex::new(CallbackState::default()), - }) - } - - #[getter] - pub(crate) fn elapsed(&self) -> f64 { - self.started.elapsed().as_secs_f64() - } - - #[pyo3(signature = (*, name, resource_kind))] - fn on_node_start( - slf: &Bound<'_, Self>, - name: String, - resource_kind: &Bound<'_, PyAny>, - ) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let callbacks: &Self = slf.get(); - if let Some(worker) = &callbacks.worker { - worker.start(py, &name, resource_kind)?; - } - if let Some(projector) = &callbacks.projector { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("resource_name", name)?; - projector - .bind(py) - .call_method("expect_resource_enrichment", (), Some(&options))?; - return Ok(()); - } - { - let mut state = callbacks.state(); - state.node_name = name; - state.node_kind = native_field(resource_kind, "value")?.extract()?; - state.sub_message = String::new(); - } - if !callbacks.is_tty { - return Ok(()); - } - callbacks.set_cursor(py, true)?; - callbacks.locked(py, || { - callbacks.state().line_active = true; - claim( - callbacks.lines.bind(py), - callbacks.stream.bind(py), - slf.as_any(), - )?; - callbacks.write_spinner_line(py) - })?; - callbacks.stop_spinner(py)?; - let (event, thread) = start_spinner(slf.as_any())?; - let mut state = callbacks.state(); - state.stop_event = Some(event.unbind()); - state.thread = Some(thread.unbind()); - Ok(()) - } - - fn on_sub_progress(&self, py: Python<'_>, message: String) -> PyResult<()> { - self.state().sub_message = message; - if self.is_tty { - self.write_spinner_line(py)?; - } - Ok(()) - } - - /// Keep a line per completed microbatch when no live spinner row shows batch progress. - fn on_batch_complete(&self, py: Python<'_>, progress: &Bound<'_, PyAny>) -> PyResult<()> { - self.write_batch_complete(py, progress) - } - - /// Report one deduplicated concurrent scheduler state transition. - fn on_scheduler_state(&self, py: Python<'_>, state: &Bound<'_, PyAny>) -> PyResult<()> { - let state: SchedulerStateText = scheduler_state(state)?; - { - let mut callback_state = self.state(); - callback_state.scheduler_count += 1; - if callback_state.scheduler_count > SCHEDULER_DIAGNOSTIC_LIMIT { - callback_state.scheduler_omitted += 1; - callback_state.last_scheduler = Some(state); - return Ok(()); - } - } - self.write_runtime_diagnostic(py, &scheduler_diagnostic_message(state)) - } - - /// Report a bounded sample of completed warehouse statement identities. - fn on_statement_complete(&self, py: Python<'_>, telemetry: &Bound<'_, PyAny>) -> PyResult<()> { - let telemetry: StatementText = statement(telemetry)?; - { - let mut state = self.state(); - let count: &mut usize = state - .query_counts - .entry(telemetry.resource_name.clone()) - .or_default(); - *count += 1; - if *count > QUERY_DIAGNOSTIC_LIMIT { - state.query_omitted += 1; - state.last_query = Some(telemetry); - return Ok(()); - } - } - self.write_runtime_diagnostic(py, &query_diagnostic_message(&telemetry)) - } - - /// Erase the live spinner row so a persistent line can take its place. - fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { - self.locked(py, || self.emit(py, CLEAR_LINE)) - } - - /// Draw the live spinner row again below persistent output. - fn redraw_transient_line(&self, py: Python<'_>) -> PyResult<()> { - self.write_spinner_line(py) - } - - fn on_node_complete(slf: &Bound<'_, Self>, node_result: &Bound<'_, PyAny>) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let callbacks: &Self = slf.get(); - if let Some(worker) = &callbacks.worker - && let Some((_, key)) = resource_identity(node_result)? - { - worker.completed(node_result, &key)?; - } - let kind: ResultKind = result_kind(node_result)?; - let mut result: Bound<'_, PyAny> = node_result.clone(); - if let Some(projector) = &callbacks.projector - && let Some((name, identity)) = resource_identity(node_result)? - { - let Some(duration_ms) = consume_terminal(projector.bind(py), &name, Some(&identity))? - else { - return Ok(()); - }; - if matches!( - kind, - ResultKind::Model | ResultKind::Seed | ResultKind::Function | ResultKind::Load - ) { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("duration_ms", duration_ms.trunc() as i64)?; - result = replaced_keywords(node_result, &options)?; - } - } - let written: PyResult<()> = Self::write_node_result(slf, &result, kind); - let evented: PyResult<()> = callbacks.write_event(py, &result); - written.and(evented) - } - - /// Write one result to the shared structured event channel. - fn write_execution_event(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { - self.write_event(py, result) - } - - fn on_check_start(&self, py: Python<'_>, name: &str) -> PyResult<()> { - if let Some(worker) = &self.worker { - worker.start_check(py, name)?; - } - Ok(()) - } - - fn on_check_complete(&self, py: Python<'_>, result: &Bound<'_, PyAny>) -> PyResult<()> { - if let Some(worker) = &self.worker { - worker.completed_check(result)?; - } - self.write_event(py, result) - } - - /// Close the optional structured execution event stream. - fn close(&self, py: Python<'_>) -> PyResult<()> { - let (scheduler, query) = { - let state = self.state(); - ( - state - .last_scheduler - .map(|last| (state.scheduler_omitted, last)), - state - .last_query - .clone() - .map(|last| (state.query_omitted, last)), - ) - }; - for line in runtime_diagnostic_summary( - scheduler, - query.as_ref().map(|(omitted, last)| (*omitted, last)), - ) { - self.write_runtime_diagnostic(py, &line)?; - } - self.event_writer.bind(py).call_method0("close")?; - Ok(()) - } - - fn _spin_until_stopped(&self, py: Python<'_>, stop_event: &Bound<'_, PyAny>) -> PyResult<()> { - while !stop_event - .call_method1("wait", (SPINNER_TICK_SECONDS,))? - .is_truthy()? - { - self.write_spinner_line(py)?; - } - Ok(()) - } -} - -/// A callable that forwards each completed microbatch to `callbacks`, built without a Python lookup. -pub(crate) fn batch_complete_callback<'py>( - callbacks: &Bound<'py, PyAny>, -) -> PyResult> { - let owned: Py = callbacks - .downcast::()? - .clone() - .unbind(); - let forward = move |arguments: &Bound<'_, PyTuple>, - _keywords: Option<&Bound<'_, PyDict>>| - -> PyResult<()> { - owned - .get() - .write_batch_complete(arguments.py(), &arguments.get_item(0)?) - }; - Ok(PyCFunction::new_closure(callbacks.py(), None, None, forward)?.into_any()) -} - -/// Forward a Python check lifecycle callback directly to its native progress owner. -pub(crate) fn check_callback<'py>( - callbacks: &Bound<'py, PyAny>, - completion: bool, -) -> PyResult> { - let owned: Py = callbacks - .downcast::()? - .clone() - .unbind(); - let forward = move |arguments: &Bound<'_, PyTuple>, - _keywords: Option<&Bound<'_, PyDict>>| - -> PyResult<()> { - let value = arguments.get_item(0)?; - if completion { - owned.get().on_check_complete(arguments.py(), &value) - } else { - owned.get().on_check_start(arguments.py(), value.extract()?) - } - }; - Ok(PyCFunction::new_closure(callbacks.py(), None, None, forward)?.into_any()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs deleted file mode 100644 index f19280d63c..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs +++ /dev/null @@ -1,47 +0,0 @@ -//! The `sqb build` completion footer. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_core::build_progress::main::format_build_footer::format_build_footer as footer; -use sqlbuild_core::build_progress::models::{BuildResultText, PythonNodeText}; -use sqlbuild_core::cli_text::models::CliStyle; - -use crate::bindings::_helpers::build_progress::result_text::{build_text, python_node_text}; - -/// Render the build completion line, warnings, failures and skipped rollup. -#[pyfunction] -#[pyo3(signature = (*, result, elapsed, use_color, python_node_results = None))] -pub(crate) fn format_build_footer( - result: &Bound<'_, PyAny>, - elapsed: f64, - use_color: bool, - python_node_results: Option<&Bound<'_, PyAny>>, -) -> PyResult { - format_build_footer_with( - &build_text(result)?, - (elapsed, use_color), - python_node_results, - Vec::new(), - ) -} - -/// The build footer, counting `checks` (Python check results) with the Python node results. -pub(crate) fn format_build_footer_with( - result: &BuildResultText, - (elapsed, use_color): (f64, bool), - python_node_results: Option<&Bound<'_, PyAny>>, - checks: Vec, -) -> PyResult { - let mut python: Vec = Vec::new(); - if let Some(results) = python_node_results { - for node in results.try_iter()? { - python.push(python_node_text(&node?)?); - } - } - python.extend(checks); - Ok(footer(result, &python, elapsed, CliStyle { use_color })) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(format_build_footer, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs index dc06fff1c1..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs @@ -1,10 +1 @@ -pub(crate) mod audit_progress; -pub(crate) mod build_callbacks; -pub(crate) mod footer; -pub(crate) mod getters; -pub(crate) mod nested_progress; -pub(crate) mod plan_facts; -pub(crate) mod progress_reporters; -pub(crate) mod progress_text; pub(crate) mod registration; -pub(crate) mod result_text; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs deleted file mode 100644 index 1220615519..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs +++ /dev/null @@ -1,405 +0,0 @@ -//! One live spinner row for all running items and grouped completed rows of `sqb test`. - -use std::sync::{Mutex, MutexGuard, PoisonError}; - -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyDict; -use pyo3::{pyclass, pymethods}; -use sqlbuild_core::build_progress::models::ErrorText; -use sqlbuild_core::cli_text::models::CliStyle; -use sqlbuild_core::nested_progress::main::render_group_header::render_group_header; -use sqlbuild_core::nested_progress::main::render_nested_item::render_nested_item; -use sqlbuild_core::nested_progress::main::render_nested_spinner_line::render_nested_spinner_line; -use sqlbuild_core::nested_progress::models::{NestedChildRow, NestedItem, NestedLayout}; - -use crate::bindings::_helpers::build_progress::result_text::text; -use sqlbuild_python_base::boundary::main::within::within; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::build_progress::constants::CLEAR_LINE; -use sqlbuild_python_base::build_progress::constants::HIDE_CURSOR; -use sqlbuild_python_base::build_progress::constants::SHOW_CURSOR; -use sqlbuild_python_base::build_progress::constants::SPINNER_FRAMES; -use sqlbuild_python_base::build_progress::constants::SPINNER_TICK_SECONDS; -use sqlbuild_python_base::build_progress::main::terminal::claim::claim; -use sqlbuild_python_base::build_progress::main::terminal::consume_terminal::consume_terminal; -use sqlbuild_python_base::build_progress::main::terminal::flush::flush; -use sqlbuild_python_base::build_progress::main::terminal::join_spinner::join_spinner; -use sqlbuild_python_base::build_progress::main::terminal::projector::projector; -use sqlbuild_python_base::build_progress::main::terminal::release::release; -use sqlbuild_python_base::build_progress::main::terminal::shared_lines::shared_lines; -use sqlbuild_python_base::build_progress::main::terminal::start_spinner::start_spinner; -use sqlbuild_python_base::build_progress::main::terminal::terminal_columns::terminal_columns; -use sqlbuild_python_base::build_progress::main::terminal::tty_flag::tty_flag; -use sqlbuild_python_base::build_progress::main::terminal::write::write; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -const MIN_NAME_WIDTH: usize = 50; -const ITEM_OPTIONS: (&[&str], &[&str]) = ( - &["group_name", "item_name"], - &[ - "status_text", - "detail", - "error_code", - "error_help", - "error_message", - "child_rows", - "canonical_resource_name", - "canonical_resource_id", - ], -); - -/// Mutable reporter state; only held briefly and never across Python calls. -#[derive(Default)] -struct ReporterState { - printed_group: Option, - running: Vec<(String, String)>, - frame: usize, - line_active: bool, - cursor_hidden: bool, - stop_event: Option>, - thread: Option>, -} - -/// Render one live spinner row for all running items and grouped completed rows. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct NestedCommandProgressCallbacks { - stream: Py, - lines: Py, - lock: Py, - label: String, - name_width: usize, - style: CliStyle, - is_tty: bool, - sequential: bool, - projector: Option>, - state: Mutex, -} - -fn optional_argument(arguments: &KeywordArguments<'_>, name: &str) -> PyResult> { - if !arguments.passed(name)? { - return Ok(None); - } - let value: Bound<'_, PyAny> = arguments.value(name)?; - if value.is_none() { - return Ok(None); - } - value.extract().map(Some) -} - -fn child_rows(arguments: &KeywordArguments<'_>) -> PyResult> { - if !arguments.passed("child_rows")? { - return Ok(Vec::new()); - } - let mut rows: Vec = Vec::new(); - for row in arguments.value("child_rows")?.try_iter()? { - let row: Bound<'_, PyAny> = row?; - rows.push(NestedChildRow { - label: text(&row, "label")?, - name: text(&row, "name")?, - status_text: text(&row, "status_text")?, - detail: text(&row, "detail")?, - }); - } - Ok(rows) -} - -impl NestedCommandProgressCallbacks { - fn state(&self) -> MutexGuard<'_, ReporterState> { - self.state.lock().unwrap_or_else(PoisonError::into_inner) - } - - fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { - within(self.lock.bind(py), |_| body()) - } - - fn layout(&self, py: Python<'_>) -> PyResult { - Ok(NestedLayout { - label: self.label.clone(), - name_width: self.name_width, - columns: terminal_columns(py)?, - }) - } - - fn draw_spinner_line(&self, py: Python<'_>) -> PyResult<()> { - let (running, frame) = { - let mut state = self.state(); - let frame: usize = state.frame; - state.frame = (frame + 1) % SPINNER_FRAMES.len(); - let running: Vec = state.running.iter().map(|(item, _)| item.clone()).collect(); - (running, frame) - }; - let stream: &Bound<'_, PyAny> = self.stream.bind(py); - write( - stream, - &render_nested_spinner_line( - &running, - SPINNER_FRAMES[frame], - &self.layout(py)?, - self.style, - ), - )?; - flush(stream) - } - - fn clear_live_row(&self, py: Python<'_>) -> PyResult<()> { - if self.is_tty && self.state().line_active { - write(self.stream.bind(py), CLEAR_LINE)?; - } - Ok(()) - } - - fn write_group_header(&self, py: Python<'_>, group: &str) -> PyResult<()> { - let previous: Option = self.state().printed_group.clone(); - if previous.as_deref() == Some(group) { - return Ok(()); - } - self.state().printed_group = Some(group.to_owned()); - let stream: &Bound<'_, PyAny> = self.stream.bind(py); - write( - stream, - &render_group_header(group, previous.is_none(), self.style), - )?; - flush(stream) - } - - fn set_cursor(&self, py: Python<'_>, hidden: bool) -> PyResult<()> { - if self.state().cursor_hidden == hidden { - return Ok(()); - } - let stream: &Bound<'_, PyAny> = self.stream.bind(py); - write(stream, if hidden { HIDE_CURSOR } else { SHOW_CURSOR })?; - flush(stream)?; - self.state().cursor_hidden = hidden; - Ok(()) - } - - /// Redraw the live row for remaining items, or retire it; return a thread to join. - fn refresh_live_row(slf: &Bound<'_, Self>) -> PyResult>> { - let py: Python<'_> = slf.py(); - let reporter: &Self = slf.get(); - if !reporter.is_tty { - return Ok(None); - } - if !reporter.state().running.is_empty() { - reporter.draw_spinner_line(py)?; - return Ok(None); - } - reporter.state().line_active = false; - release(reporter.lines.bind(py), slf.as_any())?; - reporter.set_cursor(py, false)?; - let (event, thread) = { - let mut state = reporter.state(); - (state.stop_event.take(), state.thread.take()) - }; - if let Some(event) = event { - event.bind(py).call_method0("set")?; - } - Ok(thread) - } -} - -#[pymethods] -impl NestedCommandProgressCallbacks { - #[new] - #[pyo3(signature = (**options))] - pub(crate) fn new(py: Python<'_>, options: Option>) -> PyResult { - let arguments = KeywordArguments::read( - py, - "NestedCommandProgressCallbacks", - options, - ( - &["total", "label", "stream", "use_color"], - &["name_width", "concurrency"], - ), - )?; - let stream: Bound<'_, PyAny> = arguments.value("stream")?; - let lines: Bound<'_, PyAny> = shared_lines(py)?; - let name_width: usize = if arguments.passed("name_width")? { - arguments.value("name_width")?.extract()? - } else { - MIN_NAME_WIDTH - }; - let concurrency: i64 = if arguments.passed("concurrency")? { - arguments.value("concurrency")?.extract()? - } else { - 1 - }; - Ok(Self { - is_tty: tty_flag(&stream)?, - stream: stream.unbind(), - lock: native_field(&lines, "lock")?.unbind(), - lines: lines.unbind(), - label: arguments.value("label")?.extract()?, - name_width: name_width.max(MIN_NAME_WIDTH), - style: CliStyle { - use_color: arguments.value("use_color")?.is_truthy()?, - }, - sequential: concurrency <= 1, - projector: projector(py)?, - state: Mutex::new(ReporterState::default()), - }) - } - - #[pyo3(signature = (**options))] - fn on_item_start(slf: &Bound<'_, Self>, options: Option>) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let reporter: &Self = slf.get(); - let arguments = KeywordArguments::read( - py, - "on_item_start", - options, - (&["group_name", "item_name"], &["canonical_resource_name"]), - )?; - let group: String = arguments.value("group_name")?.extract()?; - let item: String = arguments.value("item_name")?.extract()?; - if let Some(projector) = &reporter.projector { - let canonical: Option = - optional_argument(&arguments, "canonical_resource_name")?; - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item( - "resource_name", - canonical - .filter(|name| !name.is_empty()) - .unwrap_or(item.clone()), - )?; - projector - .bind(py) - .call_method("expect_resource_enrichment", (), Some(&options))?; - } - reporter.locked(py, || { - if reporter.sequential && reporter.state().running.is_empty() { - reporter.clear_live_row(py)?; - reporter.write_group_header(py, &group)?; - } - { - let mut state = reporter.state(); - match state.running.iter_mut().find(|(name, _)| *name == item) { - Some(entry) => entry.1 = group, - None => state.running.push((item, group)), - } - } - if !reporter.is_tty { - return Ok(()); - } - reporter.set_cursor(py, true)?; - reporter.state().line_active = true; - claim( - reporter.lines.bind(py), - reporter.stream.bind(py), - slf.as_any(), - )?; - reporter.draw_spinner_line(py)?; - if reporter.state().thread.is_none() { - let (event, thread) = start_spinner(slf.as_any())?; - let mut state = reporter.state(); - state.stop_event = Some(event.unbind()); - state.thread = Some(thread.unbind()); - } - Ok(()) - }) - } - - #[pyo3(signature = (**options))] - fn on_item_complete(slf: &Bound<'_, Self>, options: Option>) -> PyResult<()> { - let py: Python<'_> = slf.py(); - let reporter: &Self = slf.get(); - let arguments = KeywordArguments::read(py, "on_item_complete", options, ITEM_OPTIONS)?; - let group: String = arguments.value("group_name")?.extract()?; - let item_name: String = arguments.value("item_name")?.extract()?; - let mut detail: String = optional_argument(&arguments, "detail")?.unwrap_or_default(); - let duration_ms: Option = match &reporter.projector { - Some(projector) => { - let canonical: Option = - optional_argument(&arguments, "canonical_resource_name")?; - consume_terminal( - projector.bind(py), - &canonical - .filter(|name| !name.is_empty()) - .unwrap_or(item_name.clone()), - optional_argument(&arguments, "canonical_resource_id")?.as_deref(), - )? - } - None => None, - }; - let stopped: Option> = reporter.locked(py, || { - let item_group: String = { - let mut state = reporter.state(); - match state - .running - .iter() - .position(|(name, _)| *name == item_name) - { - Some(index) => state.running.remove(index).1, - None => group, - } - }; - if reporter.projector.is_none() || duration_ms.is_some() { - if let Some(duration_ms) = duration_ms { - detail = format!("{detail} {:.2}s", duration_ms / 1000.0); - } - let item: NestedItem = NestedItem { - item_name, - status_text: optional_argument(&arguments, "status_text")?.unwrap_or_default(), - detail, - error: ErrorText { - code: optional_argument(&arguments, "error_code")?, - message: optional_argument(&arguments, "error_message")?, - help: optional_argument(&arguments, "error_help")?, - }, - child_rows: child_rows(&arguments)?, - }; - reporter.clear_live_row(py)?; - reporter.write_group_header(py, &item_group)?; - write( - reporter.stream.bind(py), - &render_nested_item(&item, &reporter.layout(py)?, reporter.style), - )?; - } - let stopped: Option> = Self::refresh_live_row(slf)?; - flush(reporter.stream.bind(py))?; - Ok(stopped) - })?; - if let Some(thread) = stopped { - join_spinner(thread.bind(py))?; - } - Ok(()) - } - - /// Erase the live spinner row so a persistent line can take its place. - fn clear_transient_line(&self, py: Python<'_>) -> PyResult<()> { - self.locked(py, || { - let stream: &Bound<'_, PyAny> = self.stream.bind(py); - write(stream, CLEAR_LINE)?; - flush(stream) - }) - } - - /// Draw the live spinner row again below persistent output. - fn redraw_transient_line(&self, py: Python<'_>) -> PyResult<()> { - self.locked(py, || { - if self.state().line_active { - self.draw_spinner_line(py)?; - } - Ok(()) - }) - } - - fn _spin_until_stopped(&self, py: Python<'_>, stop_event: &Bound<'_, PyAny>) -> PyResult<()> { - while !stop_event - .call_method1("wait", (SPINNER_TICK_SECONDS,))? - .is_truthy()? - { - self.locked(py, || { - if !stop_event.call_method0("is_set")?.is_truthy()? && self.state().line_active { - self.draw_spinner_line(py)?; - } - Ok(()) - })?; - } - Ok(()) - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs deleted file mode 100644 index 91b3157fda..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs +++ /dev/null @@ -1,270 +0,0 @@ -//! Warehouse connection and planning phase progress reporters. - -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Mutex, PoisonError}; -use std::time::Instant; - -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyclass, pymethods}; -use sqlbuild_core::cli_text::models::CliStyle; -use sqlbuild_core::progress::main::connection_phase_line::connection_phase_line; -use sqlbuild_core::progress::main::connection_start_message::connection_start_message; -use sqlbuild_core::progress::main::is_planning_completion_message::is_planning_completion_message; -use sqlbuild_core::progress::main::is_planning_failure_message::is_planning_failure_message; - -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::build_progress::main::progress::current_projector::current_projector; -use sqlbuild_python_base::build_progress::main::progress::projector_operation_active::projector_operation_active; -use sqlbuild_python_base::build_progress::main::progress::write_persistent::write_persistent; -use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; -use sqlbuild_python_base::build_progress::models::NativeProgressProjector; - -const CONNECTION_OPTIONS: (&[&str], &[&str]) = ( - &["adapter_name", "stream"], - &[ - "blank_line_before_start", - "blank_line_after_complete", - "use_color", - ], -); -const PROJECT_COMPILE_OPERATION: &str = "project_compile"; - -/// Render simple connection start, success, and failure messages. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct ConnectionProgressReporter { - adapter_name: String, - stream: Py, - blank_line_before_start: bool, - blank_line_after_complete: bool, - style: CliStyle, -} - -impl ConnectionProgressReporter { - fn write(&self, py: Python<'_>, message: &str) -> PyResult<()> { - write_persistent(py, self.stream.bind(py), &format!("{message}\n")) - } -} - -#[pymethods] -impl ConnectionProgressReporter { - #[new] - #[pyo3(signature = (**options))] - pub(crate) fn new(py: Python<'_>, options: Option>) -> PyResult { - let arguments: KeywordArguments<'_> = KeywordArguments::read( - py, - "ConnectionProgressReporter", - options, - CONNECTION_OPTIONS, - )?; - Ok(Self { - adapter_name: arguments.value("adapter_name")?.extract()?, - stream: arguments.value("stream")?.unbind(), - blank_line_before_start: arguments.flag("blank_line_before_start")?, - blank_line_after_complete: arguments.flag("blank_line_after_complete")?, - style: CliStyle { - use_color: arguments.flag("use_color")?, - }, - }) - } - - fn on_connection_start(&self, py: Python<'_>, connection_count: i64) -> PyResult<()> { - if self.blank_line_before_start { - self.write(py, "")?; - } - let message: String = connection_start_message(&self.adapter_name, connection_count); - self.write(py, &self.style.muted(&message)) - } - - #[pyo3(signature = (*, connection_count, elapsed_seconds))] - fn on_connection_complete( - &self, - py: Python<'_>, - connection_count: i64, - elapsed_seconds: f64, - ) -> PyResult<()> { - let _ = connection_count; - let line: String = - connection_phase_line(self.style, true, (&self.adapter_name, elapsed_seconds)); - self.write(py, &line)?; - if self.blank_line_after_complete { - self.write(py, "")?; - } - Ok(()) - } - - #[pyo3(signature = (*, connection_count, elapsed_seconds))] - fn on_connection_error( - &self, - py: Python<'_>, - connection_count: i64, - elapsed_seconds: f64, - ) -> PyResult<()> { - let _ = connection_count; - let line: String = - connection_phase_line(self.style, false, (&self.adapter_name, elapsed_seconds)); - self.write(py, &line) - } -} - -/// Render simple planning phase progress messages. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct PlanningProgressReporter { - status: Py, - active: AtomicBool, - projector: Option>, -} - -impl PlanningProgressReporter { - fn call_status(&self, py: Python<'_>, method: &str, message: &str) -> PyResult<()> { - self.status.bind(py).call_method1(method, (message,))?; - Ok(()) - } - - fn call_status_keyword(&self, py: Python<'_>, method: &str, message: &str) -> PyResult<()> { - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("message", message)?; - self.status - .bind(py) - .call_method(method, (), Some(&keywords))?; - Ok(()) - } -} - -#[pymethods] -impl PlanningProgressReporter { - #[new] - #[pyo3(signature = (*, stream, use_color = false))] - pub(crate) fn new( - py: Python<'_>, - stream: &Bound<'_, PyAny>, - use_color: bool, - ) -> PyResult { - let status: Bound<'_, PyAny> = status_reporter(stream, use_color)?; - Ok(Self { - status: status.unbind(), - active: AtomicBool::new(false), - projector: current_projector(py)?, - }) - } - - pub(crate) fn on_progress(&self, py: Python<'_>, message: &str) -> PyResult<()> { - if let Some(projector) = &self.projector - && projector_operation_active(projector.get(), PROJECT_COMPILE_OPERATION)? - { - return Ok(()); - } - if is_planning_completion_message(message) { - return self.complete(py, message); - } - if is_planning_failure_message(message) { - return self.error(py, message); - } - if !self.active.load(Ordering::SeqCst) { - return self.start(py, message); - } - self.update(py, message) - } - - pub(crate) fn start(&self, py: Python<'_>, message: &str) -> PyResult<()> { - self.call_status(py, "start", message)?; - self.active.store(true, Ordering::SeqCst); - Ok(()) - } - - fn update(&self, py: Python<'_>, message: &str) -> PyResult<()> { - if !self.active.load(Ordering::SeqCst) { - return self.start(py, message); - } - self.call_status(py, "update", message) - } - - pub(crate) fn complete(&self, py: Python<'_>, message: &str) -> PyResult<()> { - self.call_status_keyword(py, "complete", message)?; - self.active.store(false, Ordering::SeqCst); - Ok(()) - } - - /// Complete with a pre-styled message written verbatim. - pub(crate) fn complete_styled(&self, py: Python<'_>, message: &str) -> PyResult<()> { - self.call_status_keyword(py, "complete_styled", message)?; - self.active.store(false, Ordering::SeqCst); - Ok(()) - } - - /// Close active progress and render a non-success terminal message. - fn error(&self, py: Python<'_>, message: &str) -> PyResult<()> { - self.call_status(py, "error", message)?; - self.active.store(false, Ordering::SeqCst); - Ok(()) - } - - #[pyo3(signature = (*, blank_line_after = false))] - pub(crate) fn finish(&self, py: Python<'_>, blank_line_after: bool) -> PyResult<()> { - self.status.bind(py).call_method0("close")?; - self.active.store(false, Ordering::SeqCst); - if blank_line_after { - self.status.bind(py).call_method0("write_blank_line")?; - } - Ok(()) - } -} - -/// Render timed `sqb dbt init` phase progress. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct DbtInitProgressReporter { - stream: Py, - status: Py, - started: AtomicBool, - phase_start: Mutex>, -} - -#[pymethods] -impl DbtInitProgressReporter { - #[new] - #[pyo3(signature = (*, stream, use_color))] - pub(crate) fn new(stream: &Bound<'_, PyAny>, use_color: bool) -> PyResult { - Ok(Self { - stream: stream.clone().unbind(), - status: status_reporter(stream, use_color)?.unbind(), - started: AtomicBool::new(false), - phase_start: Mutex::new(None), - }) - } - - fn start(&self, py: Python<'_>, message: &str) -> PyResult<()> { - if !self.started.load(Ordering::SeqCst) { - let stream: &Bound<'_, PyAny> = self.stream.bind(py); - stream.call_method1("write", ("\n",))?; - stream.call_method0("flush")?; - self.started.store(true, Ordering::SeqCst); - } - *self - .phase_start - .lock() - .unwrap_or_else(PoisonError::into_inner) = Some(Instant::now()); - self.status.bind(py).call_method1("start", (message,))?; - Ok(()) - } - - fn complete(&self, py: Python<'_>, message: &str) -> PyResult<()> { - let phase_start: Option = self - .phase_start - .lock() - .unwrap_or_else(PoisonError::into_inner) - .take(); - let elapsed_seconds: f64 = phase_start.map_or(0.0, |start| start.elapsed().as_secs_f64()); - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("message", format!("{message} ({elapsed_seconds:.2}s)"))?; - self.status - .bind(py) - .call_method("complete", (), Some(&keywords))?; - Ok(()) - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs index 36128e91a6..d55db365ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs @@ -2,18 +2,19 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::build_progress::{ - audit_progress, build_callbacks, footer, nested_progress, -}; use sqlbuild_python_base::build_progress::main::terminal::register_terminal::register_terminal; use sqlbuild_python_base::build_progress::main::transient_lines::register_transient_lines::register_transient_lines; use sqlbuild_python_base::build_progress::main::transient_status::register_transient_status::register_transient_status; +use sqlbuild_python_output::build_progress::main::register_audit_progress::register_audit_progress; +use sqlbuild_python_output::build_progress::main::register_build_callbacks::register_build_callbacks; +use sqlbuild_python_output::build_progress::main::register_footer::register_footer; +use sqlbuild_python_output::build_progress::main::register_nested_progress::register_nested_progress; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - audit_progress::register(module)?; - build_callbacks::register(module)?; - footer::register(module)?; - nested_progress::register(module)?; + register_audit_progress(module)?; + register_build_callbacks(module)?; + register_footer(module)?; + register_nested_progress(module)?; register_terminal(module)?; register_transient_lines(module)?; register_transient_status(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index f4f96ee140..7b544a29f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -3,10 +3,8 @@ use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; use sqlbuild_python_base::records::main::construct::new_error::new_error; -use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -33,6 +31,8 @@ use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_uses; use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; +use sqlbuild_python_output::build_progress::main::write_execution_header::write_execution_header; +use sqlbuild_python_output::result_output::main::results::write_execution_json_output::write_execution_json_output; use sqlbuild_python_planner::planner_entries::main::plans::build_relation_command_plan::build_relation_command_plan; use sqlbuild_python_project::graph::main::nodes::sql_ref_key::sql_ref_key; use sqlbuild_python_project::graph::main::nodes::transitive_closure_many::transitive_closure_many; @@ -49,13 +49,13 @@ use crate::bindings::_helpers::check::check_results::{ check_result_text, results_json, write_results, }; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; -use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_outputs::main::write_python_check_runtime_target::write_python_check_runtime_target; +use sqlbuild_python_output::result_output::models::ExecutionEventWriter; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_python_project::graph::main::python_nodes::python_node_names::python_node_names; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs index aef2aa2779..cc1c96e829 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs @@ -14,11 +14,11 @@ use sqlbuild_core::check_command::models::{CheckGrouping, CheckResultText, Check use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::json::models::JsonValue; -use crate::bindings::_helpers::result_output::terminal_index::current_terminal_event_index; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::build_progress::main::progress::current_native_progress_projector::current_native_progress_projector; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::result_output::main::json_value::json_value; +use sqlbuild_python_output::result_output::main::events::current_terminal_event_index::current_terminal_event_index; const WARN_SEVERITY: &str = "warn"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs index acd0e07d43..d15d9fff44 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs @@ -18,10 +18,10 @@ use sqlbuild_core::text::main::python_str_repr::python_str_repr; use crate::bindings::_helpers::cli::namespace::new_cli_namespace; use crate::bindings::_helpers::cli::surrogates::{python_text, rust_text, uses_placeholders}; -use crate::bindings::_helpers::recall::console::write_stdout_bytes; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_output::recall::main::write_stdout_bytes::write_stdout_bytes; const INT_CONVERTER: &str = "int"; const POSITIVE_INTEGER_CONVERTER: &str = "_positive_integer"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 4a5a528cbf..e5db142c99 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -28,7 +28,6 @@ use crate::bindings::_helpers::plan_command::rules::run_rules_command; use crate::bindings::_helpers::query_command::dag::run_dag_command; use crate::bindings::_helpers::query_command::query::run_query_command; use crate::bindings::_helpers::recall::command::run_recall_command; -use crate::bindings::_helpers::recall::session::InvocationRecall; use crate::bindings::_helpers::refactor::refactor_command::run_refactor_command; use crate::bindings::_helpers::review_command::command::run_review_command; use crate::bindings::_helpers::scenario_command::capture_command::run_scenario_capture_command; @@ -57,6 +56,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_compile::compile_command::main::invocation::run_compile::run_compile; use sqlbuild_python_front::discovery::models::MissingSecretsScope; +use sqlbuild_python_output::recall::models::InvocationRecall; use sqlbuild_python_project::compiled_project::main::with_job_scope::with_job_scope; use sqlbuild_python_base::boundary::main::imported::imported; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index ff3a8155a4..1032360908 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -38,15 +38,18 @@ use crate::bindings::_helpers::cost_report::attribution::take_cost_status; use crate::bindings::_helpers::diff_report::output::render_diff_error_json; use sqlbuild_python_base::native_host::models::InterruptHandler; use sqlbuild_python_base::native_host::main::interrupted::interrupted; -use crate::bindings::_helpers::output_capture::records::invocation_context; +use sqlbuild_python_output::output_capture::main::invocation_context::invocation_context; use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; -use crate::bindings::_helpers::recall::console::write_stderr; -use crate::bindings::_helpers::recall::constants::{JSON_FORMAT, RECALL_SETTINGS_CODE}; -use crate::bindings::_helpers::recall::events::escaped_text; -use crate::bindings::_helpers::recall::session::{ - InvocationRecall, effective_selection, finish_invocation_recall, start_invocation_recall, -}; -use crate::bindings::_helpers::recall::sinks::{RecallLifecycleSink, publish_lifecycle_event}; +use sqlbuild_python_output::recall::main::write_stderr::write_stderr; +use sqlbuild_python_output::recall::constants::JSON_FORMAT; +use sqlbuild_python_output::recall::constants::RECALL_SETTINGS_CODE; +use sqlbuild_python_output::recall::main::escaped_text::escaped_text; +use sqlbuild_python_output::recall::models::InvocationRecall; +use sqlbuild_python_output::recall::main::effective_selection::effective_selection; +use sqlbuild_python_output::recall::main::finish_invocation_recall::finish_invocation_recall; +use sqlbuild_python_output::recall::main::start_invocation_recall::start_invocation_recall; +use sqlbuild_python_output::recall::models::RecallLifecycleSink; +use sqlbuild_python_output::recall::main::publish_lifecycle_event::publish_lifecycle_event; use sqlbuild_python_compile::rules::constants::DEFAULT_RULES_NOTICE_ATTR; use crate::bindings::_helpers::workspace_commands::commands::maintain_sqlbuild_skills; use sqlbuild_cli::launcher::constants::LAUNCH_RECORD_OPTION; @@ -835,7 +838,7 @@ fn execute_parsed_invocation<'py>( let facts: CommandFacts = command_facts(&args)?; let policy: InvocationPolicy = invocation_policy(&facts); let recall: InvocationRecall = if policy.recalled { - start_invocation_recall(py, &args, identity, &facts)? + start_invocation_recall(py, (&args, identity, &facts), env!("CARGO_PKG_VERSION"))? } else { InvocationRecall::default() }; @@ -906,7 +909,8 @@ fn execute_parsed_invocation<'py>( Ok(code) if code.is_instance_of::() => Some(code.extract().unwrap_or(1)), Ok(_) | Err(_) => None, }; - let finished: PyResult<()> = finish_invocation_recall(py, &recall, exit_status); + let finished: PyResult<()> = + finish_invocation_recall(py, &recall, exit_status, env!("CARGO_PKG_VERSION")); let worker = recall .worker .lock() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index 7cdd01251c..84d84dc90b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -17,9 +17,6 @@ use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::event_exporting::dispatcher::EventExporterDispatcher; use crate::bindings::_helpers::native_host::setup_context::invocation_setup_context; -use crate::bindings::_helpers::recall::console::write_stderr; -use crate::bindings::_helpers::result_output::terminal_index::TerminalEventIndex; -use crate::bindings::_helpers::result_output::terminal_index::terminal_event_index_scope; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -33,6 +30,9 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_host::main::redirect_stdout_to_stderr::redirect_stdout_to_stderr; use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_output::recall::main::write_stderr::write_stderr; +use sqlbuild_python_output::result_output::main::events::terminal_event_index_scope::terminal_event_index_scope; +use sqlbuild_python_output::result_output::models::TerminalEventIndex; const OBSERVABILITY_LOGGER: &str = "sqlbuild.cli.observability"; const DEBUG_COMMAND: &str = "debug"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index a8e12a58bc..7ec1b9263e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -2,7 +2,6 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::build_progress::{progress_reporters, progress_text}; use crate::bindings::_helpers::cli::{argv, entry, namespace, observability, termination}; use crate::bindings::_helpers::{ audit_command, build_command, build_progress, change_selection, check, clone, command_runtime, @@ -14,6 +13,8 @@ use crate::bindings::_helpers::{ }; use sqlbuild_python_base::build_progress::main::progress::register_progress::register_progress; use sqlbuild_python_base::cli::main::register_launch_state::register_launch_state; +use sqlbuild_python_output::build_progress::main::register_progress_reporters::register_progress_reporters; +use sqlbuild_python_output::build_progress::main::register_progress_text::register_progress_text; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { argv::register(module)?; @@ -23,8 +24,8 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { observability::register(module)?; termination::register(module)?; register_progress(module)?; - progress_reporters::register(module)?; - progress_text::register(module)?; + register_progress_reporters(module)?; + register_progress_text(module)?; janitor::registration::register(module)?; plan_command::registration::register(module)?; freshness::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs index ab7d77a9f8..6ba7fcec77 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs @@ -13,11 +13,11 @@ use pyo3::prelude::{ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::recall::console::write_stderr; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_host::models::WakeHandler; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_output::recall::main::write_stderr::write_stderr; create_exception!( sqlbuild._native, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index e7861f767e..07bcbd44c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -20,7 +20,6 @@ use sqlbuild_core::clone_text::main::validate_clone_target_names::validate_clone use sqlbuild_core::clone_text::models::CloneItemText; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; use crate::bindings::_helpers::clone::clone_execution::{ destination_entry_count, execute_clone_plan, resource_types_by_name, }; @@ -28,8 +27,6 @@ use crate::bindings::_helpers::clone::clone_pipeline::{ClonePipeline, timed_clon use crate::bindings::_helpers::clone::clone_progress::{clone_item_text, write_line}; use crate::bindings::_helpers::command_runtime::connection::resolve_target_connection_config; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::result_output::documents::format_clone_execution_json; -use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -46,6 +43,9 @@ use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inpu use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; use sqlbuild_python_front::contracts::main::resolve_target_name::resolve_target_name; +use sqlbuild_python_output::build_progress::models::PlanningProgressReporter; +use sqlbuild_python_output::result_output::main::results::format_clone_execution_json::format_clone_execution_json; +use sqlbuild_python_output::result_output::main::results::write_execution_json_output::write_execution_json_output; use sqlbuild_python_planner::planner_entries::types::PhaseTimings; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs index 2820d658f5..5adac1f62d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -13,7 +13,6 @@ use crate::bindings::_helpers::clone::clone_command::Invocation; use crate::bindings::_helpers::clone::clone_progress::{ CloneFingerprintReporter, CloneItemPrinter, }; -use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -24,6 +23,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_output::result_output::models::ExecutionEventWriter; const DESTINATION_ENTRY_GROUPS: [&str; 4] = [ "destination_source_entries", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs index 305a7be971..397c0e2bb0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -3,8 +3,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyBool; -use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; -use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -17,6 +15,8 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::discovery::models::SecretPolicy; +use sqlbuild_python_output::build_progress::models::ConnectionProgressReporter; +use sqlbuild_python_output::build_progress::models::PlanningProgressReporter; /// The project directory, discovered inputs, adapter context, colour and progress stream. struct InvocationContext<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/mod.rs index 93879e74b9..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/mod.rs @@ -1,4 +1 @@ -pub(crate) mod conversion; -pub(crate) mod getters; pub(crate) mod registration; -pub(crate) mod storage; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/registration.rs index 6ddfa9262d..793c0b5b70 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/registration.rs @@ -1,9 +1,9 @@ //! Register the native compute log storage. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::compute_logs::storage::LocalFilesystemComputeLogStorage; +use sqlbuild_python_output::compute_logs::main::register_storage::register_storage; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() + register_storage(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index bf3309c986..77ba02c664 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -19,7 +19,7 @@ use sqlbuild_core::contract_command::models::{ ContractFindingText, ContractProgress, ContractReport, ContractResourceText, }; -use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; +use sqlbuild_python_output::build_progress::models::ConnectionProgressReporter; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_compile::compile_driver::main::setup::build_project_graph_from_compiled_project::build_project_graph_from_compiled_project; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs index de3d311170..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs @@ -1,5 +1 @@ -pub(crate) mod allocation; -pub(crate) mod health; -pub(crate) mod ledger; pub(crate) mod registration; -pub(crate) mod snowflake; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs index f9332a6bf3..fde6cdd45e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs @@ -2,13 +2,16 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::cost_telemetry::{allocation, health, ledger, snowflake}; use sqlbuild_python_front::cost_context::main::register_cost_context::register_cost_context; +use sqlbuild_python_output::cost_telemetry::main::register_allocation::register_allocation; +use sqlbuild_python_output::cost_telemetry::main::register_health::register_health; +use sqlbuild_python_output::cost_telemetry::main::register_ledger::register_ledger; +use sqlbuild_python_output::cost_telemetry::main::register_snowflake::register_snowflake; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - allocation::register(module)?; + register_allocation(module)?; register_cost_context(module)?; - health::register(module)?; - ledger::register(module)?; - snowflake::register(module) + register_health(module)?; + register_ledger(module)?; + register_snowflake(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs index 8b06e55874..89381bdc54 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs @@ -11,10 +11,10 @@ use sqlbuild_core::dbt_command::main::forwarded_dbt_args::forwarded_dbt_args; use sqlbuild_core::dbt_command::main::render_dbt_auto_init::render_dbt_auto_init; use sqlbuild_core::dbt_command::models::ForwardedArg; -use crate::bindings::_helpers::build_progress::progress_reporters::DbtInitProgressReporter; use crate::bindings::_helpers::dbt_command::init_project::build_dbt_init_project; use crate::bindings::_helpers::dbt_command::support::{color_enabled, flag, path, stream}; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_output::build_progress::models::DbtInitProgressReporter; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs index f91d1024ab..d91238bfc0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs @@ -6,7 +6,6 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::dbt_command::main::dbt_output_options::dbt_output_options; use sqlbuild_core::dbt_command::models::DbtOutputOptions; -use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; use crate::bindings::_helpers::dbt_command::auto_init::{args_tuple, ensure_twin_project}; use crate::bindings::_helpers::dbt_command::debug::run_dbt_debug; use crate::bindings::_helpers::dbt_command::support::{color_enabled, effective_dir, flag, stream}; @@ -20,6 +19,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::registry::native_attr::native_attr; use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_output::build_progress::models::PlanningProgressReporter; const PLAN_COMMAND: &str = "plan"; const DEBUG_COMMAND: &str = "debug"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs index 5f3ece3259..ad08a0423f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs @@ -11,7 +11,6 @@ use sqlbuild_core::dbt_command::main::render_dbt_init_summary::render_dbt_init_s use sqlbuild_core::dbt_command::models::DbtInitSummary; use sqlbuild_python_base::records::main::construct::new_error::new_error; -use crate::bindings::_helpers::build_progress::progress_reporters::DbtInitProgressReporter; use crate::bindings::_helpers::dbt_command::init_project::build_dbt_init_project; use crate::bindings::_helpers::dbt_command::init_project::validate_dbt_init_project; use crate::bindings::_helpers::dbt_command::support::{color_enabled, flag, path, stream}; @@ -20,6 +19,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_output::build_progress::models::DbtInitProgressReporter; fn missing_project(py: Python<'_>) -> PyErr { new_error( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs index f0528cc572..0e0e37c874 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs @@ -8,7 +8,6 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::dbt_cli::main::dbt_debug_args::dbt_debug_args; use std::time::Instant; -use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::dbt_interop::cli::resolve_dbt_config; use crate::bindings::_helpers::dbt_interop::execution::build_failed_sqlbuild_model_names; use crate::bindings::_helpers::dbt_interop::execution::build_merged_dbt_execution_argv; @@ -34,6 +33,7 @@ use sqlbuild_python_base::records::types::NativeAttr; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::plan_has_executable_work::plan_has_executable_work; use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; +use sqlbuild_python_output::build_progress::models::ConnectionProgressReporter; const PLAN_OPTIONS: (&[&str], &[&str]) = ( &["project_dir", "args"], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs index f6212e2fb0..653db671ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs @@ -4,7 +4,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::PyDict; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::dbt_interop::execution::render_dbt_execution_summary_footer; use crate::bindings::_helpers::dbt_interop::pipeline_prologue::progress; use crate::bindings::_helpers::dbt_interop::plan::render_dbt_interop_plan; @@ -26,6 +25,7 @@ use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; use sqlbuild_python_front::contracts::main::resolve_target_config::resolve_target_config; +use sqlbuild_python_output::build_progress::models::ConnectionProgressReporter; const OPTIONS: (&[&str], &[&str]) = ( &["request", "invocation"], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs index 534e087b07..aff62d6535 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs @@ -11,13 +11,6 @@ use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_host::models::{BuildExecutionResult, BuildStatus}; use crate::bindings::_helpers::build_command::policies::enforce_plan_safety_policies_binding; -use crate::bindings::_helpers::build_progress::build_callbacks::{ - BuildProgressCallbacks, batch_complete_callback, -}; -use crate::bindings::_helpers::build_progress::footer::format_build_footer_with; -use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; -use crate::bindings::_helpers::build_progress::progress_text::format_execution_header; -use crate::bindings::_helpers::build_progress::result_text::build_result_text; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -36,6 +29,12 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_outputs::main::write_runtime_target::write_runtime_target; +use sqlbuild_python_output::build_progress::main::batch_complete_callback::batch_complete_callback; +use sqlbuild_python_output::build_progress::main::build_result_text::build_result_text; +use sqlbuild_python_output::build_progress::main::format_build_footer_with::format_build_footer_with; +use sqlbuild_python_output::build_progress::main::format_execution_header::format_execution_header; +use sqlbuild_python_output::build_progress::models::BuildProgressCallbacks; +use sqlbuild_python_output::build_progress::models::ConnectionProgressReporter; const WORK_OPTIONS: (&[&str], &[&str]) = ( &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs index 5b515d2e05..f2c3a7d1e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs @@ -14,7 +14,6 @@ use sqlbuild_core::event_export::models::{EventQueue, ExportSnapshot, QueuedEven use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::event_exporting::dispatcher::{Exporter, Queued, Shared, State}; -use crate::bindings::_helpers::recall::console::write_stderr; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::event_exporting::constants::CHARGE_LIMIT_SECONDS; @@ -26,6 +25,7 @@ use sqlbuild_python_base::event_exporting::main::error_type_of::error_type_of; use sqlbuild_python_base::event_exporting::models::EventExportSummary; use sqlbuild_python_base::event_exporting::models::EventExporterFailure; use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_output::recall::main::write_stderr::write_stderr; /// Which long-lived dispatcher thread a worker runs. #[derive(Clone, Copy)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index b7661a6bed..54a5ec5f20 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -38,7 +38,6 @@ use sqlbuild_state::execution_history::constants::{ DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT, SCHEMA_VERSION, }; -use crate::bindings::_helpers::result_output::event_writer::RESULT_PATH_ENV; use sqlbuild_python_compile::compile_command::constants::REUSE_DISABLE_ENV_VAR; use sqlbuild_python_compile::compile_command::constants::TRACKED_ENVIRONMENT_PREFIXES; use sqlbuild_python_compile::compile_command::constants::UNTRACKED_ENVIRONMENT_NAMES; @@ -50,6 +49,7 @@ use sqlbuild_python_front::attachments::constants::{ MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, }; use sqlbuild_python_front::state_rows::constants::MIGRATION_WRITE_ATTEMPTS; +use sqlbuild_python_output::result_output::constants::RESULT_PATH_ENV; use sqlbuild_python_project::semantic_shapes::constants::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; /// Text constants exported under their Python names. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index f5db7771fb..78feedae01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -16,7 +16,6 @@ use sqlbuild_host::entries::models::AdapterHost; use sqlbuild_orchestration::runs::models::NodeOutcome; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; -use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; use sqlbuild_python_base::native_host::main::connection_failure::connection_failure; use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; @@ -25,6 +24,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::build_project_graph::build_project_graph; use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_output::result_output::main::results::write_execution_json_output::write_execution_json_output; use sqlbuild_python_planner::freshness::main::observe_freshness::observe_freshness; use sqlbuild_python_planner::freshness::main::read_latest::read_latest; use sqlbuild_python_planner::freshness::models::Observer; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index a6248de9eb..804ecedbca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -11,9 +11,6 @@ use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; -use crate::bindings::_helpers::build_progress::progress_reporters::{ - ConnectionProgressReporter, PlanningProgressReporter, -}; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::janitor::janitor_facts::Warehouse; use crate::bindings::_helpers::janitor::janitor_planning::{ @@ -29,6 +26,8 @@ use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_output::build_progress::models::ConnectionProgressReporter; +use sqlbuild_python_output::build_progress::models::PlanningProgressReporter; const UNKNOWN_VIEW_CODE: &str = "C503"; const UNKNOWN_VIEW_HELP: &str = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 3067773bb4..4ea2a1ffd1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -18,8 +18,6 @@ use sqlbuild_core::load_command::main::render_load_ready::render_load_ready; use sqlbuild_core::load_command::models::{LoadCounts, LoadResultText}; use sqlbuild_host::entries::models::{AdapterHost, ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; -use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::load::load_progress::{ LoadCallbackRelay, LoadProgressReporter, load_result_text, @@ -27,9 +25,6 @@ use crate::bindings::_helpers::load::load_progress::{ use crate::bindings::_helpers::load::load_selection::{ LoadProject, LoaderDefaults, loader_defaults, loader_write_schemas, target_vars, }; -use crate::bindings::_helpers::result_output::documents::format_load_execution_json; -use crate::bindings::_helpers::result_output::documents::write_execution_json_output; -use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::command_runtime::main::resolve_cli_adapter::resolve_cli_adapter; @@ -53,6 +48,11 @@ use sqlbuild_python_front::discovery::main::sources::consumer_uses::consumer_use use sqlbuild_python_front::discovery::main::sources::require_used_secrets::require_used_secrets; use sqlbuild_python_front::discovery::main::sources::sink_uses::sink_uses; use sqlbuild_python_front::discovery::models::SecretPolicy; +use sqlbuild_python_output::build_progress::main::write_execution_header::write_execution_header; +use sqlbuild_python_output::build_progress::models::ConnectionProgressReporter; +use sqlbuild_python_output::result_output::main::results::format_load_execution_json::format_load_execution_json; +use sqlbuild_python_output::result_output::main::results::write_execution_json_output::write_execution_json_output; +use sqlbuild_python_output::result_output::models::ExecutionEventWriter; use sqlbuild_python_planner::planner_resolution::main::cursor::type_cursor_overrides::type_cursor_overrides; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; use sqlbuild_python_project::references::main::render_source_relation::render_source_relation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 867deeb14e..5bdc985fbd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -28,14 +28,14 @@ use crate::bindings::_helpers::location_command::worker::{blocked_nodes, execute use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; -use crate::bindings::_helpers::recall::console::write_stderr; -use crate::bindings::_helpers::recall::session::InvocationRecall; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::compiled_phase::compiled_phase; +use sqlbuild_python_output::recall::main::write_stderr::write_stderr; +use sqlbuild_python_output::recall::models::InvocationRecall; use sqlbuild_python_project::compiled_views::models::NativeProjectView; use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationSnapshot; use sqlbuild_python_project::project_snapshot::types::NodeKeys; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs index 97af96bfd6..e48f589c3d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -20,9 +20,9 @@ use crate::bindings::_helpers::native_host::orchestration_sensors::evaluate_sens use crate::bindings::_helpers::native_host::warehouse_metadata::{ close_history_session, open_history_session, read_history, }; -use crate::bindings::_helpers::recall::console::write_stderr; use sqlbuild_python_base::boundary::main::within::within; use sqlbuild_python_base::native_host::main::redirect_stdout_to_stderr::redirect_stdout_to_stderr; +use sqlbuild_python_output::recall::main::write_stderr::write_stderr; pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResult { options.set_item("action", "publish")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index a7203be03f..e1ad555707 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -34,8 +34,6 @@ use crate::bindings::_helpers::location_command::constants::{ SQL_TEST_NODE_KIND, }; use crate::bindings::_helpers::query_command::project::ProjectAdapter; -use crate::bindings::_helpers::recall::console::write_stderr; -use crate::bindings::_helpers::recall::session::InvocationRecall; use crate::bindings::_helpers::seed_command::command::run_seed; use crate::bindings::_helpers::sql_test_command::test_command::run_test; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -47,6 +45,8 @@ use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; +use sqlbuild_python_output::recall::main::write_stderr::write_stderr; +use sqlbuild_python_output::recall::models::InvocationRecall; use sqlbuild_python_project::compiled_project::constants::SQL_TEST_KEY_PREFIX; use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationSnapshot; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/output_capture/mod.rs index d86f946270..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/output_capture/mod.rs @@ -1,6 +1 @@ -pub(crate) mod context; -pub(crate) mod dispatcher; -pub(crate) mod records; pub(crate) mod registration; -pub(crate) mod text; -pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/output_capture/registration.rs index d23a01a9bd..df555cb1ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/output_capture/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/output_capture/registration.rs @@ -2,11 +2,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::output_capture::{context, dispatcher, records, text}; +use sqlbuild_python_output::output_capture::main::register_context::register_context; +use sqlbuild_python_output::output_capture::main::register_dispatcher::register_dispatcher; +use sqlbuild_python_output::output_capture::main::register_records::register_records; +use sqlbuild_python_output::output_capture::main::register_text::register_text; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - context::register(module)?; - dispatcher::register(module)?; - records::register(module)?; - text::register(module) + register_context(module)?; + register_dispatcher(module)?; + register_records(module)?; + register_text(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 1ba70501f2..d9345949ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -11,10 +11,6 @@ use sqlbuild_core::plan_display::constants::{ }; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::build_progress::progress_reporters::{ - ConnectionProgressReporter, PlanningProgressReporter, -}; -use crate::bindings::_helpers::build_progress::progress_text::with_verbose_inspection; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::command_runtime::preview_target::validate_preview_target; use crate::bindings::_helpers::command_runtime::runtime_errors::raised; @@ -33,6 +29,9 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_base::warehouse_access::models::SyncedCopyAdapter; use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inputs::discover_project_inputs; use sqlbuild_python_front::compile_attachment::main::build_effective_settings_config::build_effective_settings_config; +use sqlbuild_python_output::build_progress::main::with_verbose_inspection::with_verbose_inspection; +use sqlbuild_python_output::build_progress::models::ConnectionProgressReporter; +use sqlbuild_python_output::build_progress::models::PlanningProgressReporter; use sqlbuild_python_planner::consumption::main::changed_policy::changed_policy; use sqlbuild_python_planner::consumption::main::policy_line::policy_line; use sqlbuild_python_planner::consumption::main::preview_changed::preview_changed; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs index 84fe6143fe..d17961f41a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs @@ -26,18 +26,27 @@ use sqlbuild_recall::store::main::resolve_invocation::resolve_invocation; use sqlbuild_recall::store::models::{OpenMode, SqliteStore}; use crate::bindings::_helpers::query_command::query::run_query_command; -use crate::bindings::_helpers::recall::clickhouse_store::{ClickHouseLocation, ClickHouseReader}; -use crate::bindings::_helpers::recall::console::{write_stderr, write_stdout_bytes}; -use crate::bindings::_helpers::recall::constants::{ - DEFAULT_MAX_CELL_BYTES, DEFAULT_QUERY_LIMIT, JSON_FORMAT, LAST_INVOCATION, - RECALL_AMBIGUOUS_CODE, RECALL_DISABLED_CODE, RECALL_NOT_FOUND_CODE, RECALL_OPTIONS, - RECALL_QUERY_CODE, RECALL_SETTINGS_CODE, RECALL_STORE_CODE, WAREHOUSE_BACKEND, - WAREHOUSE_STATE_RELATIONS_SQL, -}; use crate::bindings::_helpers::recall::rendering::{QueryContext, render_result, render_schema}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_error::new_error; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_output::recall::constants::DEFAULT_MAX_CELL_BYTES; +use sqlbuild_python_output::recall::constants::DEFAULT_QUERY_LIMIT; +use sqlbuild_python_output::recall::constants::JSON_FORMAT; +use sqlbuild_python_output::recall::constants::LAST_INVOCATION; +use sqlbuild_python_output::recall::constants::RECALL_AMBIGUOUS_CODE; +use sqlbuild_python_output::recall::constants::RECALL_DISABLED_CODE; +use sqlbuild_python_output::recall::constants::RECALL_NOT_FOUND_CODE; +use sqlbuild_python_output::recall::constants::RECALL_OPTIONS; +use sqlbuild_python_output::recall::constants::RECALL_QUERY_CODE; +use sqlbuild_python_output::recall::constants::RECALL_SETTINGS_CODE; +use sqlbuild_python_output::recall::constants::RECALL_STORE_CODE; +use sqlbuild_python_output::recall::constants::WAREHOUSE_BACKEND; +use sqlbuild_python_output::recall::constants::WAREHOUSE_STATE_RELATIONS_SQL; +use sqlbuild_python_output::recall::main::write_stderr::write_stderr; +use sqlbuild_python_output::recall::main::write_stdout_bytes::write_stdout_bytes; +use sqlbuild_python_output::recall::models::ClickHouseLocation; +use sqlbuild_python_output::recall::models::ClickHouseReader; fn user_error(py: Python<'_>, message: &str, code: &str) -> PyErr { new_error(py, ErrorId::CliUser, message, (Some(code), None)) @@ -199,7 +208,12 @@ fn open_store(py: Python<'_>, settings: &RecallSettings, checked: bool) -> PyRes config, database, .. } => ClickHouseReader::open( py, - &ClickHouseLocation::new(config, database, settings.retention_days), + &ClickHouseLocation::new( + config, + database, + settings.retention_days, + env!("CARGO_PKG_VERSION"), + ), ) .and_then(|reader| { if checked && let Err(error) = reader.require_store(py) { @@ -355,7 +369,12 @@ fn run_schema_ddl(py: Python<'_>, settings: &RecallSettings) -> PyResult { } => ( ClickHouseReader::upgrade_statements( py, - &ClickHouseLocation::new(config, database, settings.retention_days), + &ClickHouseLocation::new( + config, + database, + settings.retention_days, + env!("CARGO_PKG_VERSION"), + ), ), format!("in ClickHouse database `{database}`"), ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs index 3cf9b77191..3c939b9912 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs @@ -1,12 +1,3 @@ -pub(crate) mod clickhouse_calls; -pub(crate) mod clickhouse_store; pub(crate) mod command; -pub(crate) mod console; -pub(crate) mod constants; -pub(crate) mod events; pub(crate) mod registration; pub(crate) mod rendering; -pub(crate) mod replacements; -pub(crate) mod session; -pub(crate) mod sinks; -pub(crate) mod writer; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs index 322bb6c179..bd434ae897 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs @@ -2,9 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use crate::bindings::_helpers::recall::sinks::RecallLifecycleSink; -use crate::bindings::_helpers::recall::writer::RecallWriter; use sqlbuild_python_front::discovery::main::python_files::install_recall_writer::install_recall_writer; +use sqlbuild_python_output::recall::models::RecallLifecycleSink; +use sqlbuild_python_output::recall::models::RecallWriter; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs index ce75b9d4e7..76c8bb9004 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs @@ -9,8 +9,10 @@ use sqlbuild_recall::schema::models::Dialect; use sqlbuild_recall::store::models::{DescribedColumn, DescribedRelation, QueryResult}; use crate::bindings::_helpers::query_command::text_output::render_text; -use crate::bindings::_helpers::recall::clickhouse_calls::QueryStatistics; -use crate::bindings::_helpers::recall::constants::{JOIN_KEYS, JSON_FORMAT, NULL_TEXT}; +use sqlbuild_python_output::recall::constants::JOIN_KEYS; +use sqlbuild_python_output::recall::constants::JSON_FORMAT; +use sqlbuild_python_output::recall::constants::NULL_TEXT; +use sqlbuild_python_output::recall::models::QueryStatistics; fn truncated(text: &str, max_bytes: Option) -> String { let Some(limit) = max_bytes.filter(|limit| text.len() > *limit) else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs deleted file mode 100644 index 4c3b93c40f..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs +++ /dev/null @@ -1,178 +0,0 @@ -//! The native subscriber that publishes every lifecycle fact and artifact into Recall. - -use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, PoisonError}; - -use pyo3::exceptions::PyException; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; -use pyo3::types::PyString; -use serde_json::{Map, Value}; -use sqlbuild_host::entries::models::ExecutorHost; -use sqlbuild_recall::records::main::parse_event::parse_event; -use sqlbuild_recall::writer::models::RecallSession; - -use crate::bindings::_helpers::recall::constants::{ - INVOCATION_COMPLETED, INVOCATION_FAILED, INVOCATION_STARTED, STATEMENT_COMPLETED, - STATEMENT_FAILED, -}; -use crate::bindings::_helpers::recall::events::{escaped_text, event_object}; -use crate::bindings::_helpers::recall::replacements::RecallReplacements; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; - -/// `event` as the JSON object `lifecycle_event_to_json` encodes; `None` when it cannot be encoded. -pub(crate) fn lifecycle_event_object( - py: Python<'_>, - event: &Bound<'_, PyAny>, -) -> PyResult>> { - if let Some(object) = event_object(event)? { - return Ok(Some(object)); - } - let encoded = host_entry(py, ExecutorHost::LifecycleEventToJson)?.call1((event,))?; - let text = escaped_text(encoded.downcast::()?)?; - match parse_event(&text) { - Ok(object) => Ok(Some(object)), - Err(_unencodable) => Ok(None), - } -} - -/// Queue `event` in `session`; an event that cannot be encoded is skipped. -pub(crate) fn publish_lifecycle_event( - py: Python<'_>, - session: &RecallSession, - event: &Bound<'_, PyAny>, -) -> PyResult<()> { - if let Some(object) = lifecycle_event_object(py, event)? { - session.publish_event_object(object); - } - Ok(()) -} - -/// The SQL and resource of a statement that has not reached its terminal event yet. -type RunningSql = HashMap)>; - -/// Which of an invocation's own lifecycle events the sink has recorded, so an interrupted run records each exactly once. -#[derive(Debug, Default)] -pub(crate) struct InvocationEvents { - started: AtomicBool, - ended: AtomicBool, -} - -impl InvocationEvents { - /// Whether `invocation_started` was recorded. - pub(crate) fn started(&self) -> bool { - self.started.load(Ordering::SeqCst) - } - - /// Whether `invocation_completed` or `invocation_failed` was recorded. - pub(crate) fn ended(&self) -> bool { - self.ended.load(Ordering::SeqCst) - } - - fn note(&self, event_type: &str) { - if event_type == INVOCATION_STARTED { - self.started.store(true, Ordering::SeqCst); - } else if event_type == INVOCATION_COMPLETED || event_type == INVOCATION_FAILED { - self.ended.store(true, Ordering::SeqCst); - } - } -} - -/// Publishes lifecycle events, through the Recall replacement sink on the delivery thread when one is configured, and SQL artifacts of the active session; delivery is best effort. The SQL of a statement that fails or is cancelled is queued again with its terminal event's priority. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct RecallLifecycleSink { - pub(crate) session: Arc, - pub(crate) replacements: Arc, - pub(crate) running_sql: Arc>, - pub(crate) invocation_events: Arc, -} - -#[pymethods] -impl RecallLifecycleSink { - /// Record `event`; an event Recall cannot encode is skipped, never raised into the command. - fn __call__(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { - recording(py, self.publish(py, event)) - } - - /// Record one SQL artifact; text Recall cannot encode is skipped, never raised into the command. - #[pyo3(signature = (*, sql, statement_id=None, resource_id=None))] - fn record_sql( - &self, - py: Python<'_>, - sql: &Bound<'_, PyString>, - statement_id: Option<&Bound<'_, PyString>>, - resource_id: Option<&Bound<'_, PyString>>, - ) -> PyResult<()> { - let texts = (|| -> PyResult<(String, Option, Option)> { - Ok(( - escaped_text(sql)?, - statement_id.map(escaped_text).transpose()?, - resource_id.map(escaped_text).transpose()?, - )) - })(); - recording( - py, - texts.map(|(sql, statement_id, resource_id)| { - self.session - .publish_sql(&sql, statement_id.as_deref(), resource_id.as_deref()); - if let Some(statement_id) = statement_id { - self.running().insert(statement_id, (sql, resource_id)); - } - }), - ) - } -} - -impl RecallLifecycleSink { - fn running(&self) -> std::sync::MutexGuard<'_, RunningSql> { - self.running_sql - .lock() - .unwrap_or_else(PoisonError::into_inner) - } - - /// Forget a statement's SQL at its terminal event; a failed one's SQL is queued again first. - fn settle_statement(&self, event: &Bound<'_, PyAny>, event_type: &str) -> PyResult<()> { - if event_type != STATEMENT_COMPLETED && event_type != STATEMENT_FAILED { - return Ok(()); - } - let Some(statement_id) = event.getattr_opt("statement_id")? else { - return Ok(()); - }; - if statement_id.is_none() { - return Ok(()); - } - let statement_id: String = escaped_text(&statement_id.str()?)?; - let Some((sql, resource_id)) = self.running().remove(&statement_id) else { - return Ok(()); - }; - if event_type == STATEMENT_FAILED { - self.session - .publish_failed_sql(&sql, Some(&statement_id), resource_id.as_deref()); - } - Ok(()) - } - - fn publish(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { - let event_type: String = match event.getattr_opt("event_type")? { - Some(event_type) => event_type.str()?.to_string(), - None => String::new(), - }; - self.settle_statement(event, &event_type)?; - if !self - .replacements - .deliver_lifecycle(py, &self.session, event)? - { - publish_lifecycle_event(py, &self.session, event)?; - } - self.invocation_events.note(&event_type); - Ok(()) - } -} - -/// `outcome` with ordinary exceptions dropped, so recording never changes what the command raises; interrupts still propagate. -fn recording(py: Python<'_>, outcome: PyResult<()>) -> PyResult<()> { - match outcome { - Err(error) if error.is_instance_of::(py) => Ok(()), - other => other, - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs deleted file mode 100644 index b80f25b1ba..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs +++ /dev/null @@ -1,146 +0,0 @@ -//! `sqlbuild.recall.RecallWriter`: what a Recall replacement sink publishes through. - -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Mutex, PoisonError}; - -use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; -use pyo3::types::{PyBytes, PyBytesMethods}; -use serde_json::{Map, Value}; -use sqlbuild_host::entries::models::ExecutorHost; - -use crate::bindings::_helpers::recall::sinks::lifecycle_event_object; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::records::main::fields::field::field as native_field; - -/// The one stream a writer publishes to. -#[derive(Debug)] -enum Target { - /// The events the sink published for the one it was handed. - Lifecycle(Mutex>>), - /// The chunk the sink was handed, by sequence, and what it published for it. - Output { - sequence: u64, - published: Mutex>>, - }, -} - -/// Publishes the records a Recall replacement sink keeps; valid only while the sink call runs. -#[pyclass(module = "sqlbuild._native", name = "RecallWriter", frozen)] -pub(crate) struct RecallWriter { - target: Target, - open: AtomicBool, -} - -impl RecallWriter { - /// A writer for one lifecycle event handed to the lifecycle replacement. - pub(crate) fn lifecycle() -> Self { - Self::new(Target::Lifecycle(Mutex::new(Vec::new()))) - } - - /// A writer for the output chunk with `sequence` handed to the output replacement. - pub(crate) fn output(sequence: u64) -> Self { - Self::new(Target::Output { - sequence, - published: Mutex::new(None), - }) - } - - fn new(target: Target) -> Self { - Self { - target, - open: AtomicBool::new(true), - } - } - - /// End the sink call: later publishing raises, and the events or output bytes published are returned. - pub(crate) fn finish(&self) -> (Vec>, Option>) { - self.open.store(false, Ordering::SeqCst); - match &self.target { - Target::Lifecycle(published) => ( - std::mem::take(&mut *published.lock().unwrap_or_else(PoisonError::into_inner)), - None, - ), - Target::Output { published, .. } => ( - Vec::new(), - published - .lock() - .unwrap_or_else(PoisonError::into_inner) - .take(), - ), - } - } - - fn ensure_open(&self) -> PyResult<()> { - if self.open.load(Ordering::SeqCst) { - return Ok(()); - } - Err(PyValueError::new_err( - "a RecallWriter can publish only while the sink call it was passed to runs", - )) - } -} - -#[pymethods] -impl RecallWriter { - /// Record `event` in Recall; for the lifecycle replacement sink. - fn publish_lifecycle(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { - self.ensure_open()?; - let Target::Lifecycle(published) = &self.target else { - return Err(PyValueError::new_err( - "this RecallWriter belongs to a command-output sink; use publish_command_output", - )); - }; - if !event.is_instance(&host_entry(py, ExecutorHost::LifecycleEvent)?)? { - return Err(PyTypeError::new_err( - "publish_lifecycle expects a LifecycleEvent", - )); - } - if let Some(object) = lifecycle_event_object(py, event)? { - published - .lock() - .unwrap_or_else(PoisonError::into_inner) - .push(object); - } - Ok(()) - } - - /// Record `record.data` in Recall as the output chunk the sink was passed; at most once per call. - fn publish_command_output(&self, py: Python<'_>, record: &Bound<'_, PyAny>) -> PyResult<()> { - self.ensure_open()?; - let Target::Output { - sequence, - published, - } = &self.target - else { - return Err(PyValueError::new_err( - "this RecallWriter belongs to a lifecycle-event sink; use publish_lifecycle", - )); - }; - if !record.is_instance(&host_entry(py, ExecutorHost::CommandOutputRecord)?)? { - return Err(PyTypeError::new_err( - "publish_command_output expects a CommandOutputRecord", - )); - } - let record_sequence: u64 = native_field(record, "sequence")?.extract()?; - if record_sequence != *sequence { - return Err(PyValueError::new_err(format!( - "publish_command_output expects the record passed to this call (sequence \ - {sequence}) or a copy made with record.replace(), not sequence {record_sequence}" - ))); - } - let data: Vec = native_field(record, "data")? - .downcast::() - .map_err(|_| PyTypeError::new_err("record.data must be bytes"))? - .as_bytes() - .to_vec(); - let mut slot = published.lock().unwrap_or_else(PoisonError::into_inner); - if slot.is_some() { - return Err(PyValueError::new_err( - "a command-output record can be published once per sink call", - )); - } - *slot = Some(data); - Ok(()) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index 412cfbcf3b..183e6f0d32 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -68,10 +68,7 @@ use crate::bindings::_helpers::record_behaviours::scopes::{ use crate::bindings::_helpers::record_behaviours::values::{ finish_fingerprint_set, logical_type_display_name, sql_value_kind, }; -use crate::bindings::_helpers::{ - build_progress, compute_logs, dbt_events, dbt_interop, event_exporting, execution_history, - result_output, -}; +use crate::bindings::_helpers::{dbt_events, dbt_interop, event_exporting, execution_history}; use sqlbuild_python_base::records::models::{GetterEntry, RecordBehaviours}; use sqlbuild_python_base::records::types::ClassMethod; use sqlbuild_python_base::records::types::ClassProperty; @@ -358,11 +355,11 @@ pub(crate) const BEHAVIOUR_ORDERED: &[&str] = &[ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ sqlbuild_python_compile::analysis::constants::GETTERS, sqlbuild_python_compile::analysis_session::constants::GETTERS, - build_progress::getters::GETTERS, + sqlbuild_python_output::build_progress::constants::GETTERS, sqlbuild_python_compile::compile_command::constants::GETTERS, sqlbuild_python_base::compile_context::constants::GETTERS, sqlbuild_python_project::compiled_views::constants::GETTERS, - compute_logs::getters::GETTERS, + sqlbuild_python_output::compute_logs::constants::GETTERS, dbt_events::getters::GETTERS, dbt_interop::getters::GETTERS, sqlbuild_python_base::diagnostics::constants::GETTERS, @@ -376,7 +373,7 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ sqlbuild_python_planner::planner_phases::constants::GETTERS, sqlbuild_python_base::presentation_text::constants::GETTERS, sqlbuild_python_compile::project_assembly::constants::GETTERS, - result_output::getters::GETTERS, + sqlbuild_python_output::result_output::constants::GETTERS, sqlbuild_python_compile::rules::constants::GETTERS, sqlbuild_python_front::scopes::constants::GETTERS, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs deleted file mode 100644 index 3ff857802d..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs +++ /dev/null @@ -1,954 +0,0 @@ -//! Execution-result JSON documents for build, seed, load, clone, test, audit and scenario commands. - -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_host::entries::models::ExecutorHost; -use sqlbuild_host::models::{ - AuditExecutionResult, AuditOutcome, BuildExecutionResult, BuildStatus, ExecutionStatus, - LoadExecutionResult, ScenarioLocalRunStatus, ScenarioRunResult, SeedExecutionResult, - SqlTestExecutionResult, SqlTestOutcome, -}; - -use crate::bindings::_helpers::result_output::aggregation::RetainedBuildResults; -use crate::bindings::_helpers::result_output::assets::{ - clone_asset, function_assets, load_assets, model_assets, python_node_assets, seed_assets, - without_none, -}; -use crate::bindings::_helpers::result_output::checks::{ - audit_checks, python_checks, scenario_check, scenario_result, snapshot_result, sql_test_checks, -}; -use crate::bindings::_helpers::result_output::projection::{Projection, rounded}; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::compile_outputs::main::indented_json::indented_json; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_base::native_host::types::ReadResult; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_planner::plan_output::main::any::any; -use sqlbuild_python_planner::plan_output::main::dict::dict; -use sqlbuild_python_planner::plan_output::main::items::items; -use sqlbuild_python_planner::plan_output::main::text::text; - -const JSON_VERSION: i64 = 1; -const FAILED: &str = "failed"; -const SUCCESS: &str = "success"; -const SKIPPED: &str = "skipped"; -const BUILD_OPTIONS: (&[&str], &[&str]) = ( - &["result", "plan"], - &[ - "python_node_results", - "python_check_results", - "command", - "run_id", - "cost", - ], -); -const DEGRADABLE_BUILD_OPTIONS: (&[&str], &[&str]) = ( - &["result", "plan", "execution_succeeded"], - &[ - "python_node_results", - "python_check_results", - "command", - "run_id", - "cost", - ], -); -const TYPED_BUILD_OPTIONS: (&[&str], &[&str]) = (&["plan", "execution_succeeded"], BUILD_OPTIONS.1); -const SCENARIO_OPTIONS: (&[&str], &[&str]) = ( - &["results"], - &["local", "run_namespace", "namespace_source", "duration_ms"], -); -const SNAPSHOT_OPTIONS: (&[&str], &[&str]) = ( - &["results"], - &["refresh", "run_namespace", "namespace_source"], -); -/// The optional sections of a document after `checks`. -#[derive(Default)] -struct Extras<'py> { - scenarios: Vec>, - run_id: Option>, - cost: Option>, - execution: Option>, - /// Entries appended after every other section. - trailing: Vec<(&'static str, Bound<'py, PyAny>)>, -} - -/// The document body shared by every command. -struct Document<'py> { - command: String, - status: String, - summary: Bound<'py, PyDict>, - assets: Vec>, - checks: Vec>, -} - -/// `json.dumps(payload, indent=2) + "\n"`, failed when the lifecycle terminal failed. -fn emitted<'py>( - projection: &Projection<'py>, - document: Document<'py>, - extras: Extras<'py>, -) -> PyResult { - let py: Python<'py> = document.summary.py(); - let mut status: String = document.status; - if let Some(terminal) = projection.lifecycle_terminal()? - && text(&native_field(&terminal, "event_type")?)?.ends_with(FAILED) - { - status = FAILED.to_owned(); - } - let payload: Bound<'py, PyDict> = dict( - py, - vec![ - ("version", any(py, JSON_VERSION)?), - ("command", any(py, document.command)?), - ("status", any(py, status)?), - ("summary", document.summary.into_any()), - ("assets", PyTuple::new(py, document.assets)?.into_any()), - ("checks", PyTuple::new(py, document.checks)?.into_any()), - ], - )?; - if !extras.scenarios.is_empty() { - payload.set_item("scenarios", PyTuple::new(py, extras.scenarios)?)?; - } - if let Some(run_id) = extras.run_id.filter(|run_id| !run_id.is_none()) { - payload.set_item("run_id", run_id)?; - } - if let Some(cost) = extras.cost.filter(|cost| !cost.is_none()) { - payload.set_item("cost", cost)?; - } - if let Some(execution) = extras.execution { - payload.set_item("execution", execution)?; - } - for (key, value) in extras.trailing { - payload.set_item(key, value)?; - } - Ok(indented_json(payload.as_any())? + "\n") -} - -fn count_status(results: &[Bound<'_, PyAny>], field: &str, status: &str) -> PyResult { - let mut count: usize = 0; - for result in results { - if native_field(&native_field(result, field)?, "value")?.eq(status)? { - count += 1; - } - } - Ok(count) -} - -fn count_truthy(results: &[Bound<'_, PyAny>], field: &str) -> PyResult { - let mut count: usize = 0; - for result in results { - if native_field(result, field)?.is_truthy()? { - count += 1; - } - } - Ok(count) -} - -fn summary<'py>(py: Python<'py>, entries: &[(&str, usize)]) -> PyResult> { - dict( - py, - entries - .iter() - .map(|(key, count)| Ok((*key, any(py, *count)?))) - .collect::>()?, - ) -} - -fn outcome_status(fail_count: usize) -> String { - if fail_count == 0 { SUCCESS } else { FAILED }.to_owned() -} - -/// The argument when it was passed and is not `None`. -fn passed<'py>( - arguments: &KeywordArguments<'py>, - name: &str, -) -> PyResult>> { - let value: Bound<'py, PyAny> = arguments.value(name)?; - Ok((!value.is_none()).then_some(value)) -} - -/// A sequence argument that defaults to empty. -fn sequence<'py>(py: Python<'py>, value: Option>) -> Bound<'py, PyAny> { - value.unwrap_or_else(|| PyTuple::empty(py).into_any()) -} - -fn count(value: i64) -> PyResult { - usize::try_from(value).map_err(|_| { - pyo3::exceptions::PyOverflowError::new_err("can't convert negative int to unsigned") - }) -} - -/// Format build command execution results as JSON. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn format_build_execution_json<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult { - let arguments: KeywordArguments<'py> = - KeywordArguments::read(py, "format_build_execution_json", options, BUILD_OPTIONS)?; - let result = BuildExecutionResult::read(&arguments.value("result")?)?; - build_json(py, &arguments, &result, Vec::new()) -} - -/// The audit-history projection counts, flagged as degraded when any write failed. -pub(crate) fn audit_projection<'py>( - py: Python<'py>, - (attempted, written, failed): (i64, i64, i64), -) -> PyResult)>> { - let counts: Vec<(&str, Bound<'py, PyAny>)> = [ - ("attempted_count", attempted), - ("written_count", written), - ("failed_count", failed), - ] - .into_iter() - .map(|(key, value)| Ok((key, any(py, value)?))) - .collect::>()?; - let failed: bool = counts[2].1.is_truthy()?; - let mut entries: Vec<(&'static str, Bound<'py, PyAny>)> = vec![( - "audit_result_projection", - dict(py, counts.clone())?.into_any(), - )]; - if failed { - let mut reason: Vec<(&str, Bound<'py, PyAny>)> = - vec![("reason", any(py, "audit_result_projection_failure")?)]; - reason.extend(counts); - entries.push(("projection_degraded", any(py, true)?)); - entries.push(( - "projection_degradation_reasons", - PyList::new(py, [dict(py, reason)?])?.into_any(), - )); - } - Ok(entries) -} - -/// The build document with its audit-history projection, or a minimal degraded document. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn format_degradable_build_execution_json<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "format_degradable_build_execution_json", - options, - DEGRADABLE_BUILD_OPTIONS, - )?; - let result = BuildExecutionResult::read(&arguments.value("result")?)?; - degradable_build_json(py, &arguments, &result) -} - -/// Production build completion already owns the converted executor result. -pub(crate) fn format_build_execution_json_with<'py>( - py: Python<'py>, - result: &BuildExecutionResult>, - options: Option>, -) -> PyResult { - let arguments = KeywordArguments::read( - py, - "format_build_execution_json", - options, - TYPED_BUILD_OPTIONS, - )?; - degradable_build_json(py, &arguments, result) -} - -fn degradable_build_json<'py>( - py: Python<'py>, - arguments: &KeywordArguments<'py>, - result: &BuildExecutionResult>, -) -> PyResult { - let execution_succeeded: bool = arguments.flag("execution_succeeded")?; - let rendered: PyResult = audit_projection( - py, - ( - result.audit_result_projection_attempted_count, - result.audit_result_projection_written_count, - result.audit_result_projection_failed_count, - ), - ) - .and_then(|trailing| build_json(py, arguments, result, trailing)); - let error: pyo3::PyErr = match rendered { - Ok(payload) => return Ok(payload), - Err(error) => error, - }; - let degradable: bool = error.is_instance_of::(py) - || error.is_instance_of::(py) - || error.is_instance( - py, - &host_entry(py, ExecutorHost::ObservabilityValidationError)?, - ); - if !degradable { - return Err(error); - } - let command: String = if arguments.passed("command")? { - arguments.value("command")?.extract()? - } else { - "build".to_owned() - }; - degraded_document( - py, - (command, if execution_succeeded { SUCCESS } else { FAILED }), - error.value(py).as_any(), - execution_succeeded, - ) -} - -fn build_json<'py>( - py: Python<'py>, - arguments: &KeywordArguments<'py>, - result: &BuildExecutionResult>, - trailing: Vec<(&'static str, Bound<'py, PyAny>)>, -) -> PyResult { - let plan: Bound<'py, PyAny> = arguments.value("plan")?; - let command: String = if arguments.passed("command")? { - arguments.value("command")?.extract()? - } else { - "build".to_owned() - }; - let projection: Projection<'py> = Projection::installed(py)?; - let python_node_results: Bound<'py, PyAny> = - sequence(py, passed(arguments, "python_node_results")?); - let python_check_results: Bound<'py, PyAny> = - sequence(py, passed(arguments, "python_check_results")?); - let retained = RetainedBuildResults::new(result, &projection)?; - let nodes: Vec> = projection.terminal_results(&python_node_results)?; - let checks_run: Vec> = projection.terminal_results(&python_check_results)?; - let partial = retained.is_partial(result) - || nodes.len() != python_node_results.len()? - || checks_run.len() != python_check_results.len()?; - let python_fail: usize = count_status(&nodes, "status", FAILED)?; - let python_success: usize = count_status(&nodes, "status", SUCCESS)?; - let python_skipped: usize = count_status(&nodes, "status", SKIPPED)?; - let check_fail: usize = count_truthy(&checks_run, "failed")?; - let check_warn: usize = count_truthy(&checks_run, "warned")?; - let check_pass: usize = count_truthy(&checks_run, "passed")?; - let mut python_warnings: usize = 0; - for warned in nodes.iter().chain(&checks_run) { - python_warnings += native_field(warned, "warning_messages")?.len()?; - } - python_warnings += retained - .loads - .iter() - .map(|result| result.warning_messages.len()) - .sum::(); - let status: String = - if result.status == BuildStatus::Failed || python_fail > 0 || check_fail > 0 { - FAILED.to_owned() - } else { - SUCCESS.to_owned() - }; - let mut checks: Vec> = sql_test_checks(&projection, &retained.tests)?; - for audits in [&retained.source_audits, &retained.end_audits] { - checks.extend(audit_checks(&projection, audits, false)?); - } - for model in &retained.models { - checks.extend(audit_checks( - &projection, - &model.audit_results.iter().collect::>(), - projection.terminal_recorded(model)?, - )?); - } - checks.extend(python_checks(&projection, &checks_run)?); - let mut assets: Vec> = - model_assets(&projection, &retained.models, Some(&plan))?; - assets.extend(seed_assets(&projection, &retained.seeds, Some(&plan))?); - assets.extend(function_assets( - &projection, - &retained.functions, - Some(&plan), - )?); - assets.extend(load_assets(&projection, &retained.loads)?); - assets.extend(python_node_assets(&projection, &nodes)?); - let counted: [usize; 5] = if partial { - let counts = retained.counts(); - [ - counts.success, - counts.failure, - counts.timeout, - counts.skipped, - counts.warning, - ] - } else { - [ - count(result.success_count)?, - count(result.failure_count)?, - count(result.timeout_count)?, - count(result.skipped_count)?, - count(result.warning_count)?, - ] - }; - let summary: Bound<'py, PyDict> = summary( - py, - &[ - ("success_count", counted[0] + python_success), - ("failure_count", counted[1] + python_fail + check_fail), - ("timeout_count", counted[2]), - ("skipped_count", counted[3] + python_skipped), - ("warning_count", counted[4] + check_warn + python_warnings), - ("python_check_pass_count", check_pass), - ("python_check_warn_count", check_warn), - ("python_check_fail_count", check_fail), - ], - )?; - emitted( - &projection, - Document { - command, - status, - summary, - assets, - checks, - }, - Extras { - run_id: passed(arguments, "run_id")?, - cost: passed(arguments, "cost")?, - trailing, - ..Extras::default() - }, - ) -} - -/// Format seed command execution results as JSON. -#[pyfunction] -#[pyo3(signature = (*, results, plan))] -pub(crate) fn format_seed_execution_json<'py>( - py: Python<'py>, - results: Bound<'py, PyAny>, - plan: Bound<'py, PyAny>, -) -> PyResult { - let projection: Projection<'py> = Projection::installed(py)?; - let all = Vec::::read(&results)?; - let results = projection.retained(&all)?; - let fail_count = results - .iter() - .filter(|result| result.status == ExecutionStatus::Failed) - .count(); - emitted( - &projection, - Document { - command: "seed".to_owned(), - status: outcome_status(fail_count), - summary: summary( - py, - &[ - ( - "success_count", - results - .iter() - .filter(|result| result.status == ExecutionStatus::Success) - .count(), - ), - ("failure_count", fail_count), - ("total_count", results.len()), - ], - )?, - assets: seed_assets(&projection, &results, Some(&plan))?, - checks: Vec::new(), - }, - Extras::default(), - ) -} - -/// Format load command execution results as JSON. -#[pyfunction] -#[pyo3(signature = (*, results))] -pub(crate) fn format_load_execution_json<'py>( - py: Python<'py>, - results: Bound<'py, PyAny>, -) -> PyResult { - let projection: Projection<'py> = Projection::installed(py)?; - let all = Vec::>::read(&results)?; - let results = projection.retained(&all)?; - let fail_count = results - .iter() - .filter(|result| result.status == ExecutionStatus::Failed) - .count(); - emitted( - &projection, - Document { - command: "load".to_owned(), - status: outcome_status(fail_count), - summary: summary( - py, - &[ - ( - "success_count", - results - .iter() - .filter(|result| result.status == ExecutionStatus::Success) - .count(), - ), - ("failure_count", fail_count), - ( - "skipped_count", - results - .iter() - .filter(|result| result.status == ExecutionStatus::Skipped) - .count(), - ), - ("total_count", results.len()), - ], - )?, - assets: load_assets(&projection, &results)?, - checks: Vec::new(), - }, - Extras::default(), - ) -} - -/// Format direct clone command execution results as JSON. -#[pyfunction] -#[pyo3(signature = (*, result, resource_types_by_name, phase_timings = None))] -pub(crate) fn format_clone_execution_json<'py>( - py: Python<'py>, - result: Bound<'py, PyAny>, - resource_types_by_name: Bound<'py, PyAny>, - phase_timings: Option>, -) -> PyResult { - let projection: Projection<'py> = Projection::installed(py)?; - let mut assets: Vec> = Vec::new(); - let (mut failure_count, mut warning_count): (usize, usize) = (0, 0); - for item in items(&native_field(&result, "item_results")?)? { - let name: String = text(&native_field(&item, "name")?)?; - let resource_type: String = text(&resource_types_by_name.get_item(&name)?)?; - if !projection.resource_terminal_recorded(&name, &format!("{resource_type}:{name}"))? { - continue; - } - let status: Bound<'py, PyAny> = native_field(&native_field(&item, "status")?, "value")?; - failure_count += usize::from(status.eq(FAILED)?); - warning_count += usize::from(status.eq("warning")?); - assets.push(clone_asset(&projection, &item, &resource_type)?); - } - let total: usize = assets.len(); - emitted( - &projection, - Document { - command: "clone".to_owned(), - status: outcome_status(failure_count), - summary: summary( - py, - &[ - ("success_count", total - failure_count - warning_count), - ("failure_count", failure_count), - ("warning_count", warning_count), - ("total_count", total), - ], - )?, - assets, - checks: Vec::new(), - }, - Extras { - trailing: phase_timings - .into_iter() - .map(|timings| ("phase_timings", timings.into_any())) - .collect(), - ..Extras::default() - }, - ) -} - -/// Format test command execution results as JSON. -#[pyfunction] -#[pyo3(signature = (*, results))] -pub(crate) fn format_test_execution_json<'py>( - py: Python<'py>, - results: Bound<'py, PyAny>, -) -> PyResult { - let all = Vec::>::read(&results)?; - test_execution_json(py, &all) -} - -/// The test command's document from the executor's already-converted results. -pub(crate) fn test_execution_json<'py>( - py: Python<'py>, - all: &[SqlTestExecutionResult>], -) -> PyResult { - let projection: Projection<'py> = Projection::installed(py)?; - let results = projection.retained(all)?; - let pass_count = results - .iter() - .filter(|result| result.outcome == SqlTestOutcome::Pass) - .count(); - let timeout_count = results - .iter() - .filter(|result| result.outcome == SqlTestOutcome::Timeout) - .count(); - let fail_count: usize = results.len() - pass_count; - emitted( - &projection, - Document { - command: "test".to_owned(), - status: outcome_status(fail_count), - summary: summary( - py, - &[ - ("pass_count", pass_count), - ("fail_count", fail_count), - ("timeout_count", timeout_count), - ("total_count", results.len()), - ], - )?, - assets: Vec::new(), - checks: sql_test_checks(&projection, &results)?, - }, - Extras::default(), - ) -} - -/// Format audit command execution results as JSON. -#[pyfunction] -#[pyo3(signature = (*, results, configured_concurrency = None, worker_count = None))] -pub(crate) fn format_audit_execution_json<'py>( - py: Python<'py>, - results: Bound<'py, PyAny>, - configured_concurrency: Option>, - worker_count: Option>, -) -> PyResult { - let all = Vec::>::read(&results)?; - audit_execution_json(py, &all, (configured_concurrency, worker_count), Vec::new()) -} - -/// Audit reporting from already-converted executor results, before the document is serialized. -pub(crate) fn audit_execution_json<'py>( - py: Python<'py>, - all: &[AuditExecutionResult>], - (configured_concurrency, worker_count): (Option>, Option>), - trailing: Vec<(&'static str, Bound<'py, PyAny>)>, -) -> PyResult { - let configured_concurrency: Bound<'py, PyAny> = - configured_concurrency.map_or_else(|| any(py, 1_i64), Ok)?; - let projection: Projection<'py> = Projection::installed(py)?; - let results = projection.retained(all)?; - let fail_count = results - .iter() - .filter(|result| result.outcome == AuditOutcome::Error) - .count(); - let workers: Bound<'py, PyAny> = match worker_count.filter(|count| !count.is_none()) { - Some(workers) => workers, - None => py - .import("builtins")? - .getattr("min")? - .call1((&configured_concurrency, results.len()))?, - }; - emitted( - &projection, - Document { - command: "audit".to_owned(), - status: outcome_status(fail_count), - summary: summary( - py, - &[ - ( - "pass_count", - results - .iter() - .filter(|result| result.outcome == AuditOutcome::Pass) - .count(), - ), - ( - "warn_count", - results - .iter() - .filter(|result| result.outcome == AuditOutcome::Warn) - .count(), - ), - ("fail_count", fail_count), - ("total_count", results.len()), - ], - )?, - assets: Vec::new(), - checks: audit_checks(&projection, &results, false)?, - }, - Extras { - execution: Some(dict( - py, - vec![ - ("configured_concurrency", configured_concurrency), - ("worker_count", workers), - ], - )?), - trailing, - ..Extras::default() - }, - ) -} - -fn namespace_execution<'py>( - py: Python<'py>, - run_namespace: Option>, - namespace_source: Bound<'py, PyAny>, -) -> PyResult> { - dict( - py, - vec![ - ( - "scenario_namespace", - run_namespace.unwrap_or_else(|| py.None().into_bound(py)), - ), - ("scenario_namespace_source", namespace_source), - ], - ) -} - -fn scenario_failed(result: &ScenarioRunResult>, local: bool) -> bool { - if !local { - return result.status == ExecutionStatus::Failed; - } - matches!( - result.local_status, - Some(ScenarioLocalRunStatus::Fail | ScenarioLocalRunStatus::Error) - ) -} - -/// Format scenario test command execution results as JSON. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn format_scenario_execution_json<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "format_scenario_execution_json", - options, - SCENARIO_OPTIONS, - )?; - let results = Vec::>::read(&arguments.value("results")?)?; - scenario_json(py, &arguments, &results) -} - -/// Production scenario completion already owns its converted executor results. -pub(crate) fn format_scenario_execution_json_with<'py>( - py: Python<'py>, - results: &[ScenarioRunResult>], - options: Option>, -) -> PyResult { - let arguments = KeywordArguments::read( - py, - "format_scenario_execution_json", - options, - (&[], SCENARIO_OPTIONS.1), - )?; - scenario_json(py, &arguments, results) -} - -fn scenario_json<'py>( - py: Python<'py>, - arguments: &KeywordArguments<'py>, - results: &[ScenarioRunResult>], -) -> PyResult { - let local: bool = arguments.passed("local")? && arguments.flag("local")?; - let run_namespace: Option> = passed(arguments, "run_namespace")?; - let namespace_source: Option> = passed(arguments, "namespace_source")?; - let duration_ms: Option = arguments.value("duration_ms")?.extract()?; - let mut projection: Projection<'py> = Projection::installed(py)?; - let mut kept: Vec<&ScenarioRunResult>> = Vec::new(); - for result in results { - let name = &result.scenario_name; - if projection.resource_terminal_recorded(name, &format!("sql_scenario:{name}"))? { - kept.push(result); - } - } - let mut fail_count: usize = 0; - let (mut assets, mut checks, mut scenarios) = (Vec::new(), Vec::new(), Vec::new()); - for result in &kept { - fail_count += usize::from(scenario_failed(result, local)); - projection.scenario = Some(result.scenario_name.clone()); - assets.extend(seed_assets( - &projection, - &result.seed_results.iter().collect::>(), - None, - )?); - assets.extend(function_assets( - &projection, - &result.function_results.iter().collect::>(), - None, - )?); - assets.extend(model_assets( - &projection, - &result.model_results.iter().collect::>(), - None, - )?); - projection.scenario = None; - checks.push(scenario_check(py, result)?); - scenarios.push(scenario_result(py, result)?); - } - let mut summary: Bound<'py, PyDict> = summary( - py, - &[ - ("pass_count", kept.len() - fail_count), - ("fail_count", fail_count), - ("total_count", kept.len()), - ], - )?; - if let Some(duration) = duration_ms { - summary = without_none( - py, - vec![ - ("pass_count", any(py, kept.len() - fail_count)?), - ("fail_count", any(py, fail_count)?), - ("total_count", any(py, kept.len())?), - ("duration_ms", rounded(py, duration)?), - ], - )?; - } - emitted( - &projection, - Document { - command: "scenario test".to_owned(), - status: outcome_status(fail_count), - summary, - assets, - checks, - }, - Extras { - scenarios, - execution: Some(namespace_execution( - py, - run_namespace, - namespace_source.map_or_else(|| any(py, "unset"), Ok)?, - )?), - ..Extras::default() - }, - ) -} - -/// Format scenario snapshot sync/refresh execution results as JSON. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn format_scenario_snapshot_execution_json<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "format_scenario_snapshot_execution_json", - options, - SNAPSHOT_OPTIONS, - )?; - let results: Bound<'py, PyAny> = arguments.value("results")?; - let refresh: bool = arguments.passed("refresh")? && arguments.flag("refresh")?; - let run_namespace: Option> = passed(&arguments, "run_namespace")?; - let namespace_source: Option> = passed(&arguments, "namespace_source")?; - let projection: Projection<'py> = Projection::installed(py)?; - let results: Vec> = items(&results)?; - let fail_count: usize = count_status(&results, "status", FAILED)?; - emitted( - &projection, - Document { - command: if refresh { - "scenario snapshot refresh" - } else { - "scenario snapshot sync" - } - .to_owned(), - status: outcome_status(fail_count), - summary: summary( - py, - &[ - ("pass_count", results.len() - fail_count), - ("fail_count", fail_count), - ("total_count", results.len()), - ], - )?, - assets: Vec::new(), - checks: Vec::new(), - }, - Extras { - scenarios: results - .iter() - .map(snapshot_result) - .collect::>()?, - execution: Some(namespace_execution( - py, - run_namespace, - namespace_source.map_or_else(|| any(py, "unset"), Ok)?, - )?), - ..Extras::default() - }, - ) -} - -/// Warn about a failed result projection, then format a minimal valid document. -fn degraded_document( - py: Python<'_>, - (command, status): (String, &str), - error: &Bound<'_, PyAny>, - execution_succeeded: bool, -) -> PyResult { - let completion: &str = if execution_succeeded { - "warehouse execution completed successfully" - } else { - "execution result remains authoritative" - }; - let stderr: Bound<'_, PyAny> = py.import("sys")?.getattr("stderr")?; - stderr.call_method1( - "write", - (format!( - "warning: integration output degraded: {}; {completion}\n", - error.str()? - ),), - )?; - stderr.call_method0("flush")?; - let payload: Bound<'_, PyDict> = dict( - py, - vec![ - ("version", any(py, JSON_VERSION)?), - ("command", any(py, command)?), - ("status", any(py, status)?), - ("summary", PyDict::new(py).into_any()), - ("assets", PyList::empty(py).into_any()), - ("checks", PyList::empty(py).into_any()), - ("projection_degraded", any(py, true)?), - ], - )?; - Ok(indented_json(payload.as_any())? + "\n") -} - -/// Write execution JSON to stdout or a requested side-channel file. -#[pyfunction] -#[pyo3(signature = (*, payload, json_output, json_output_path))] -pub(crate) fn write_execution_json_output( - py: Python<'_>, - payload: Bound<'_, PyAny>, - json_output: bool, - json_output_path: Bound<'_, PyAny>, -) -> PyResult<()> { - if !json_output_path.is_none() { - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("parents", true)?; - keywords.set_item("exist_ok", true)?; - native_field(&json_output_path, "parent")?.call_method("mkdir", (), Some(&keywords))?; - let encoding: Bound<'_, PyDict> = PyDict::new(py); - encoding.set_item("encoding", "utf-8")?; - json_output_path.call_method("write_text", (payload,), Some(&encoding))?; - return Ok(()); - } - if json_output { - let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; - stdout.call_method1("write", (payload,))?; - stdout.call_method0("flush")?; - } - Ok(()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(format_build_execution_json, module)?)?; - module.add_function(wrap_pyfunction!(format_seed_execution_json, module)?)?; - module.add_function(wrap_pyfunction!(format_load_execution_json, module)?)?; - module.add_function(wrap_pyfunction!(format_clone_execution_json, module)?)?; - module.add_function(wrap_pyfunction!(format_test_execution_json, module)?)?; - module.add_function(wrap_pyfunction!(format_audit_execution_json, module)?)?; - module.add_function(wrap_pyfunction!(format_scenario_execution_json, module)?)?; - module.add_function(wrap_pyfunction!( - format_scenario_snapshot_execution_json, - module - )?)?; - module.add_function(wrap_pyfunction!( - format_degradable_build_execution_json, - module - )?)?; - module.add_function(wrap_pyfunction!(write_execution_json_output, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs deleted file mode 100644 index 22b2bf3d8c..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs +++ /dev/null @@ -1,250 +0,0 @@ -//! Append canonical integration-result records for a parent integration, one per claimed terminal. - -use std::sync::Mutex; - -use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyDict; -use pyo3::{pyclass, pymethods}; -use sqlbuild_core::integration_results::errors::IntegrationRecordError; -use sqlbuild_core::terminal_events::models::TerminalRequest; - -use crate::bindings::_helpers::result_output::integration_records::{ - clone_record, degraded_record, degraded_warning, execution_succeeded, integration_record, -}; -use crate::bindings::_helpers::result_output::projection::resource_identity; -use crate::bindings::_helpers::result_output::terminal_index::{ - TerminalEventClaim, TerminalEventIndex, current_index_object, -}; -use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::result_output::types::Checked; - -pub(crate) const RESULT_PATH_ENV: &str = "SQLBUILD_INTEGRATION_RESULT_PATH"; -const BUILD_OPTIONS: (&[&str], &[&str]) = (&["result", "plan"], &["command"]); - -/// Thread-safe JSON Lines execution event writer. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct ExecutionEventWriter { - /// A `threading.Lock`; it releases the GIL while it waits. - lock: Py, - terminal_index: Option>, - /// The open text stream, until the writer is closed. - stream: Mutex>>, -} - -/// A Python context manager entered for the duration of `body`. -fn within(manager: &Bound<'_, PyAny>, body: impl FnOnce() -> PyResult) -> PyResult { - manager.call_method0("__enter__")?; - let outcome: PyResult = body(); - let none = manager.py().None(); - manager.call_method1("__exit__", (&none, &none, &none))?; - outcome -} - -impl ExecutionEventWriter { - fn open_stream(&self, py: Python<'_>) -> PyResult>> { - Ok(self - .stream - .lock() - .map_err(|_| PyRuntimeError::new_err("execution event writer lock is poisoned"))? - .as_ref() - .map(|stream| stream.clone_ref(py))) - } - - /// Run `body` holding the writer lock and the index's output scope, when the stream is open. - fn serialized( - &self, - py: Python<'_>, - body: impl FnOnce(&Bound<'_, PyAny>) -> PyResult<()>, - ) -> PyResult<()> { - within(self.lock.bind(py), || { - let Some(stream) = self.open_stream(py)? else { - return Ok(()); - }; - match &self.terminal_index { - Some(index) => within(&index.get().output_lock(py), || body(stream.bind(py))), - None => body(stream.bind(py)), - } - }) - } - - fn claim( - &self, - py: Python<'_>, - request: &TerminalRequest, - ) -> PyResult> { - match &self.terminal_index { - Some(index) => index.get().claim(py, request), - None => Ok(None), - } - } - - /// Write the record, or after a contract failure warn and write the degraded record. - fn publish( - stream: &Bound<'_, PyAny>, - terminal: &Bound<'_, PyAny>, - record: Checked>, - degraded: impl FnOnce() -> Checked>, - ) -> PyResult<()> { - let error: IntegrationRecordError = match record? { - Ok(Some(line)) => return write(stream, &line), - Ok(None) => return Ok(()), - Err(error) => error, - }; - let stderr: Bound<'_, PyAny> = stream.py().import("sys")?.getattr("stderr")?; - stderr.call_method1( - "write", - (degraded_warning(&error, execution_succeeded(terminal)?),), - )?; - stderr.call_method0("flush")?; - match degraded()? { - Ok(Some(line)) => write(stream, &line), - Ok(None) | Err(_) => Ok(()), - } - } -} - -fn write(stream: &Bound<'_, PyAny>, line: &str) -> PyResult<()> { - stream.call_method1("write", (line,))?; - stream.call_method0("flush")?; - Ok(()) -} - -#[pymethods] -impl ExecutionEventWriter { - #[new] - #[pyo3(signature = (*, path = None))] - pub(crate) fn new(py: Python<'_>, path: Option>) -> PyResult { - let configured: Bound<'_, PyAny> = py - .import("os")? - .getattr("environ")? - .call_method1("get", (RESULT_PATH_ENV,))?; - let path: Option> = - match path.filter(|path| path.is_truthy().unwrap_or(true)) { - Some(path) => Some(path), - None if configured.is_truthy()? => Some( - py.import("pathlib")? - .getattr("Path")? - .call1((configured,))?, - ), - None => None, - }; - let stream: Option> = match path { - Some(path) => { - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("parents", true)?; - keywords.set_item("exist_ok", true)?; - native_field(&path, "parent")?.call_method("mkdir", (), Some(&keywords))?; - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("mode", "a")?; - options.set_item("encoding", "utf-8")?; - Some(path.call_method("open", (), Some(&options))?.unbind()) - } - None => None, - }; - Ok(Self { - lock: py.import("threading")?.call_method0("Lock")?.unbind(), - terminal_index: current_index_object(py)?, - stream: Mutex::new(stream), - }) - } - - /// Write canonical terminal evidence plus typed build-result enrichment. - #[pyo3(signature = (**options))] - fn write_build_result( - &self, - py: Python<'_>, - options: Option>, - ) -> PyResult<()> { - let arguments: KeywordArguments<'_> = - KeywordArguments::read(py, "write_build_result", options, BUILD_OPTIONS)?; - let result: Bound<'_, PyAny> = arguments.value("result")?; - let plan: Bound<'_, PyAny> = arguments.value("plan")?; - let command: String = if arguments.passed("command")? { - arguments.value("command")?.extract()? - } else { - "build".to_owned() - }; - self.serialized(py, |stream| { - let Some((resource_name, resource_id)) = resource_identity(&result)? else { - return Ok(()); - }; - let Some(claim) = self.claim( - py, - &TerminalRequest { - resource_name, - resource_id: Some(resource_id), - ..TerminalRequest::default() - }, - )? - else { - return Ok(()); - }; - let terminal: Bound<'_, PyAny> = claim.terminal_event(py); - let sequence: usize = claim.sequence(); - Self::publish( - stream, - &terminal, - integration_record((&result, &plan), (&terminal, sequence), &command), - || degraded_record(&result, (&terminal, sequence), &command), - ) - }) - } - - /// Write canonical terminal evidence plus typed clone-result enrichment. - #[pyo3(signature = (*, item, resource_type))] - fn write_clone_result( - &self, - py: Python<'_>, - item: Bound<'_, PyAny>, - resource_type: &str, - ) -> PyResult<()> { - self.serialized(py, |stream| { - let name: String = native_field(&item, "name")?.extract()?; - let Some(claim) = self.claim( - py, - &TerminalRequest { - resource_id: Some(format!("{resource_type}:{name}")), - resource_name: name, - ..TerminalRequest::default() - }, - )? - else { - return Ok(()); - }; - let terminal: Bound<'_, PyAny> = claim.terminal_event(py); - let sequence: usize = claim.sequence(); - Self::publish( - stream, - &terminal, - Ok(clone_record(&item, (&terminal, sequence))?.map(Some)), - || degraded_record(&item, (&terminal, sequence), "clone"), - ) - }) - } - - /// Close the event stream when configured. - fn close(&self, py: Python<'_>) -> PyResult<()> { - within(self.lock.bind(py), || { - let Some(stream) = self.open_stream(py)? else { - return Ok(()); - }; - let close = || -> PyResult<()> { - *self.stream.lock().map_err(|_| { - PyRuntimeError::new_err("execution event writer lock is poisoned") - })? = None; - stream.bind(py).call_method0("close")?; - Ok(()) - }; - match &self.terminal_index { - Some(index) => within(&index.get().output_lock(py), close), - None => close(), - } - }) - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs index 2a86603b89..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/mod.rs @@ -1,12 +1 @@ -pub(crate) mod aggregation; -pub(crate) mod assets; -pub(crate) mod checks; -pub(crate) mod documents; -pub(crate) mod event_writer; -pub(crate) mod getters; -pub(crate) mod integration_envelope; -pub(crate) mod integration_records; -pub(crate) mod projection; pub(crate) mod registration; -pub(crate) mod resource_ids; -pub(crate) mod terminal_index; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs index b94c4fbf40..688e362a7d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/registration.rs @@ -2,15 +2,18 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::result_output::{ - aggregation, documents, event_writer, integration_envelope, resource_ids, terminal_index, -}; +use sqlbuild_python_output::result_output::main::events::register_event_writer::register_event_writer; +use sqlbuild_python_output::result_output::main::events::register_integration_envelope::register_integration_envelope; +use sqlbuild_python_output::result_output::main::events::register_resource_ids::register_resource_ids; +use sqlbuild_python_output::result_output::main::events::register_terminal_index::register_terminal_index; +use sqlbuild_python_output::result_output::main::results::register_aggregation::register_aggregation; +use sqlbuild_python_output::result_output::main::results::register_documents::register_documents; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - terminal_index::register(module)?; - resource_ids::register(module)?; - aggregation::register(module)?; - integration_envelope::register(module)?; - event_writer::register(module)?; - documents::register(module) + register_terminal_index(module)?; + register_resource_ids(module)?; + register_aggregation(module)?; + register_integration_envelope(module)?; + register_event_writer(module)?; + register_documents(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs deleted file mode 100644 index f9f29a2ca4..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs +++ /dev/null @@ -1,309 +0,0 @@ -//! The invocation-scoped index of canonical lifecycle facts that result output projects from. - -use sqlbuild_python_base::build_progress::main::progress::terminal_request::terminal_request; -use std::sync::{Mutex, MutexGuard}; - -use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, -}; -use pyo3::types::{PyDict, PyInt, PyString, PyTuple}; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; -use sqlbuild_core::terminal_events::main::claim_resource_terminal::claim_resource_terminal; - -use sqlbuild_core::terminal_events::main::consume_terminal_fact::consume_terminal_fact; -use sqlbuild_core::terminal_events::main::latest_resource_terminal::latest_resource_terminal; -use sqlbuild_core::terminal_events::main::lifecycle_terminal::lifecycle_terminal; -use sqlbuild_core::terminal_events::models::{TerminalFact, TerminalIndex, TerminalRequest}; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_base::records::main::registry::native_attr::native_attr; -use sqlbuild_python_base::records::types::NativeAttr; - -/// The module attribute holding the `ContextVar` of the installed index. -const CONTEXT_ATTRIBUTE: &str = "_terminal_event_index_context"; -const CONTEXT_NAME: &str = "sqlbuild_terminal_event_index"; - -struct IndexState { - index: TerminalIndex, - events: Vec>, -} - -/// Retain canonical facts and claim resource terminals for result projection. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct TerminalEventIndex { - state: Mutex, - output_lock: Py, -} - -/// One claimed terminal with its immutable canonical publication sequence. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct TerminalEventClaim { - #[pyo3(get)] - pub(crate) terminal: Py, - #[pyo3(get)] - pub(crate) event_sequence: usize, -} - -/// `payload.get(key)` when it is a string. -fn payload_text(payload: &Bound<'_, PyAny>, key: &str) -> PyResult> { - let value: Bound<'_, PyAny> = mapping_get(payload, key)?; - if value.is_instance_of::() { - return Ok(Some(value.extract()?)); - } - Ok(None) -} - -/// The payload attempt number, when it is an `int` (or `bool`, which Python compares as one). -fn payload_attempt_number(payload: &Bound<'_, PyAny>) -> PyResult> { - let value: Bound<'_, PyAny> = mapping_get(payload, "attempt_number")?; - if !value.is_instance_of::() { - return Ok(None); - } - let Ok(number) = value.extract::() else { - return Ok(None); - }; - Ok(Some(number)) -} - -fn optional_text(event: &Bound<'_, PyAny>, name: &str) -> PyResult> { - native_field(event, name)?.extract() -} - -/// The facts of one lifecycle event that terminal indexing reads. -fn terminal_fact(event: &Bound<'_, PyAny>) -> PyResult { - let payload: Bound<'_, PyAny> = native_field(event, "payload")?; - Ok(TerminalFact { - event_id: native_field(event, "event_id")?.extract()?, - event_type: native_field(event, "event_type")?.extract()?, - resource_id: optional_text(event, "resource_id")?, - resource_name: payload_text(&payload, "resource_name")?, - resource_attempt_id: optional_text(event, "resource_attempt_id")?, - attempt_number: payload_attempt_number(&payload)?, - operation_name: payload_text(&payload, "operation_name")?, - }) -} - -impl TerminalEventIndex { - fn locked(&self) -> PyResult> { - self.state - .lock() - .map_err(|_| PyRuntimeError::new_err("terminal event index lock is poisoned")) - } - - fn event_at(&self, py: Python<'_>, sequence: Option) -> PyResult>> { - let state: MutexGuard<'_, IndexState> = self.locked()?; - Ok(sequence.map(|sequence| state.events[sequence].clone_ref(py))) - } - - /// The latest terminal of a resource without claiming it. - pub(crate) fn latest_terminal<'py>( - &self, - py: Python<'py>, - resource_name: &str, - resource_id: Option<&str>, - ) -> PyResult>> { - let sequence: Option = { - let state: MutexGuard<'_, IndexState> = self.locked()?; - latest_resource_terminal(&state.index, resource_name, resource_id) - }; - Ok(self - .event_at(py, sequence)? - .map(|event| event.into_bound(py))) - } - - /// The strongest aggregate terminal for the final result document. - pub(crate) fn aggregate_terminal<'py>( - &self, - py: Python<'py>, - ) -> PyResult>> { - let sequence: Option = lifecycle_terminal(&self.locked()?.index); - Ok(self - .event_at(py, sequence)? - .map(|event| event.into_bound(py))) - } - - /// Claim the terminal a result callback reports on. - pub(crate) fn claim( - &self, - py: Python<'_>, - request: &TerminalRequest, - ) -> PyResult> { - let mut state: MutexGuard<'_, IndexState> = self.locked()?; - let (index, claimed) = claim_resource_terminal(std::mem::take(&mut state.index), request); - state.index = index; - let Some(sequence) = claimed else { - return Ok(None); - }; - Ok(Some(TerminalEventClaim { - terminal: state.events[sequence].clone_ref(py), - event_sequence: sequence, - })) - } - - /// The reentrant lock that serializes claims with physical result output. - pub(crate) fn output_lock<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { - self.output_lock.bind(py).clone() - } -} - -#[pymethods] -impl TerminalEventIndex { - #[new] - pub(crate) fn new(py: Python<'_>) -> PyResult { - Ok(Self { - state: Mutex::new(IndexState { - index: TerminalIndex::default(), - events: Vec::new(), - }), - output_lock: py.import("threading")?.call_method0("RLock")?.unbind(), - }) - } - - /// Store a known fact once in dispatcher publication order. - fn consume(&self, event: &Bound<'_, PyAny>) -> PyResult<()> { - let fact: TerminalFact = terminal_fact(event)?; - let mut state: MutexGuard<'_, IndexState> = self.locked()?; - let (index, stored) = consume_terminal_fact(std::mem::take(&mut state.index), fact); - state.index = index; - if stored.is_some() { - state.events.push(event.clone().unbind()); - } - Ok(()) - } - - /// Claim the latest exact terminal available when a result callback arrives. - #[pyo3(signature = (**options))] - fn claim_resource_terminal( - &self, - py: Python<'_>, - options: Option>, - ) -> PyResult> { - self.claim( - py, - &terminal_request(py, "claim_resource_terminal", options)?, - ) - } - - /// Return the latest matching terminal without claiming it. - #[pyo3(signature = (*, resource_name, resource_id = None))] - fn resource_terminal<'py>( - &self, - py: Python<'py>, - resource_name: &str, - resource_id: Option<&str>, - ) -> PyResult>> { - self.latest_terminal(py, resource_name, resource_id) - } - - /// Return the strongest aggregate terminal available to final JSON. - fn lifecycle_terminal<'py>(&self, py: Python<'py>) -> PyResult>> { - self.aggregate_terminal(py) - } - - /// Serialize terminal claims with physical integration-result output. - fn output_serialization_scope<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { - self.output_lock(py) - } - - /// Return idempotent canonical facts in publication order. - fn events<'py>(&self, py: Python<'py>) -> PyResult> { - let state: MutexGuard<'_, IndexState> = self.locked()?; - PyTuple::new(py, state.events.iter().map(|event| event.bind(py))) - } -} - -/// Installs one index for the duration of a `with` block. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct TerminalEventIndexScope { - index: Py, - token: Mutex>>, -} - -fn context_variable(py: Python<'_>) -> PyResult> { - native_attr(py, NativeAttr::TerminalEventIndexContext) -} - -#[pymethods] -impl TerminalEventIndexScope { - fn __enter__(&self, py: Python<'_>) -> PyResult> { - let token: Bound<'_, PyAny> = context_variable(py)?.call_method1("set", (&self.index,))?; - *self.token.lock().map_err(|_| { - PyRuntimeError::new_err("terminal event index scope lock is poisoned") - })? = Some(token.unbind()); - Ok(self.index.clone_ref(py)) - } - - #[pyo3(signature = (*_exc_info))] - fn __exit__(&self, py: Python<'_>, _exc_info: &Bound<'_, PyTuple>) -> PyResult { - let token: Option> = self - .token - .lock() - .map_err(|_| PyRuntimeError::new_err("terminal event index scope lock is poisoned"))? - .take(); - if let Some(token) = token { - context_variable(py)?.call_method1("reset", (token,))?; - } - Ok(false) - } -} - -/// The terminal event index installed for this invocation. -pub(crate) fn current_index(py: Python<'_>) -> PyResult>> { - let installed: Bound<'_, PyAny> = context_variable(py)?.call_method0("get")?; - if installed.is_none() { - return Ok(None); - } - Ok(Some(installed.extract()?)) -} - -/// The installed terminal event index, as an owned reference. -pub(crate) fn current_index_object(py: Python<'_>) -> PyResult>> { - let installed: Bound<'_, PyAny> = context_variable(py)?.call_method0("get")?; - if installed.is_none() { - return Ok(None); - } - Ok(Some(installed.extract()?)) -} - -impl TerminalEventClaim { - pub(crate) fn terminal_event<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { - self.terminal.bind(py).clone() - } - - pub(crate) fn sequence(&self) -> usize { - self.event_sequence - } -} - -/// Return the terminal event index installed for this invocation. -#[pyfunction] -pub(crate) fn current_terminal_event_index(py: Python<'_>) -> PyResult> { - context_variable(py)?.call_method0("get") -} - -/// Install and reliably restore one invocation terminal event index. -#[pyfunction] -pub(crate) fn terminal_event_index_scope(index: Py) -> TerminalEventIndexScope { - TerminalEventIndexScope { - index, - token: Mutex::new(None), - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - let py: Python<'_> = module.py(); - let keywords: Bound<'_, PyDict> = PyDict::new(py); - pyo3::types::PyDictMethods::set_item(&keywords, "default", py.None())?; - module.add( - CONTEXT_ATTRIBUTE, - py.import("contextvars")? - .getattr("ContextVar")? - .call((CONTEXT_NAME,), Some(&keywords))?, - )?; - module.add_class::()?; - module.add_class::()?; - module.add_class::()?; - module.add_function(wrap_pyfunction!(current_terminal_event_index, module)?)?; - module.add_function(wrap_pyfunction!(terminal_event_index_scope, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/command.rs index 3131e9b442..7bdc4355bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/command.rs @@ -25,7 +25,6 @@ use sqlbuild_analysis::review_command::models::{ use sqlbuild_core::git_revisions::main::head_commit::head_commit; use crate::bindings::_helpers::clone::clone_progress::write_line; -use crate::bindings::_helpers::recall::console::{write_stderr, write_stdout_bytes}; use crate::bindings::_helpers::review_command::comparison::{ ComparisonContext, Comparisons, Sides, compare_models, prepare_sides, }; @@ -45,6 +44,8 @@ use crate::bindings::_helpers::review_command::support::{ error_parts, failed_error, invocation_id, phase, plan_error, progress_stream, utc_now_compact, }; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_output::recall::main::write_stderr::write_stderr; +use sqlbuild_python_output::recall::main::write_stdout_bytes::write_stdout_bytes; const SUCCESS_STATUS: &str = "success"; const SKIPPED_STATUS: &str = "skipped"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs index c977e40b26..1e0d9c4c48 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs @@ -14,7 +14,6 @@ use sqlbuild_core::scenario_command::models::ScenarioNamespace; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_host::models::ScenarioRunResult; -use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::scenario_command::conversions::{ capture_row, scenario_result_row, write_local_runtime_target, write_warehouse_runtime_target, }; @@ -27,6 +26,7 @@ use sqlbuild_python_base::build_progress::main::transient_status::status_reporte use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_output::build_progress::models::ConnectionProgressReporter; /// How a finished scenario is reported. #[derive(Clone, Debug)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index c7cd050861..39f4ef7598 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -22,8 +22,6 @@ use sqlbuild_core::scenario_command::models::{ }; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; -use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use sqlbuild_python_base::boundary::main::call_with::call_with; @@ -42,6 +40,8 @@ use sqlbuild_python_compile::compile_driver::main::stages::discover_project_inpu use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_front::contracts::main::resolve_effective_scenario_config::resolve_effective_scenario_config; use sqlbuild_python_front::discovery::main::failures::sql_analysis_guidance::sql_analysis_guidance; +use sqlbuild_python_output::build_progress::main::write_execution_header::write_execution_header; +use sqlbuild_python_output::build_progress::models::PlanningProgressReporter; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; const METADATA_MODULE: &str = "importlib.metadata"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs index 5b6ccc98dd..98998b7fa8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs @@ -8,9 +8,6 @@ use sqlbuild_core::scenario_command::main::snapshot_capture_warning::snapshot_ca use sqlbuild_core::scenario_command::models::ScenarioNamespace; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::build_progress::progress_reporters::PlanningProgressReporter; -use crate::bindings::_helpers::result_output::documents::format_scenario_snapshot_execution_json; -use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::scenario_command::conversions::succeeded; use crate::bindings::_helpers::scenario_command::presentation::{ CompletionKind, RelayContext, SectionStatus, begin_section, connection_hooks, progress_relay, @@ -24,6 +21,9 @@ use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_output::build_progress::models::PlanningProgressReporter; +use sqlbuild_python_output::result_output::main::results::format_scenario_snapshot_execution_json::format_scenario_snapshot_execution_json; +use sqlbuild_python_output::result_output::main::results::write_execution_json_output::write_execution_json_output; /// The outcome of syncing local snapshots: its exit code and capture results. pub(crate) struct LocalSync<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index 197f8556d9..74ddc70cbd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -25,8 +25,6 @@ use crate::bindings::_helpers::change_selection::selection::{ ChangedProject, changed_project, tests_and_models, with_since_section, }; use crate::bindings::_helpers::native_host::statement_report::{StatementReport, StatementSource}; -use crate::bindings::_helpers::result_output::documents::format_scenario_execution_json_with; -use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::scenario_command::conversions::scenario_result_row; use crate::bindings::_helpers::scenario_command::presentation::{ CompletionKind, RelayContext, SectionStatus, activity, begin_section, connection_hooks, @@ -46,6 +44,8 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_host::types::ReadResult; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_output::result_output::main::results::format_scenario_execution_json_with::format_scenario_execution_json_with; +use sqlbuild_python_output::result_output::main::results::write_execution_json_output::write_execution_json_output; const LOCAL_ADAPTER: &str = "duckdb"; const LOCAL_STATUSES: [&str; 4] = ["PASS", "FAIL", "ERROR", "SKIP"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 9a8f3cf4b8..588f117eb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -16,15 +16,10 @@ use sqlbuild_core::seed_command::main::seed_exit_code::seed_exit_code; use sqlbuild_core::seed_command::models::SeedRow; use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; -use crate::bindings::_helpers::build_progress::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::invocation::resolve_command_invocation; use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; -use crate::bindings::_helpers::result_output::documents::format_seed_execution_json; -use crate::bindings::_helpers::result_output::documents::write_execution_json_output; -use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::build_progress::main::progress::current_native_progress_projector::current_native_progress_projector; @@ -36,6 +31,11 @@ use sqlbuild_python_base::records::main::fields::replaced_keywords::replaced_key use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::build_static_pipeline_result::build_static_pipeline_result; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_output::build_progress::main::write_execution_header::write_execution_header; +use sqlbuild_python_output::build_progress::models::ConnectionProgressReporter; +use sqlbuild_python_output::result_output::main::results::format_seed_execution_json::format_seed_execution_json; +use sqlbuild_python_output::result_output::main::results::write_execution_json_output::write_execution_json_output; +use sqlbuild_python_output::result_output::models::ExecutionEventWriter; use sqlbuild_python_planner::planner_entries::main::plans::build_seed_command_plan::build_seed_command_plan; use sqlbuild_python_planner::planner_entries::main::plans::read_selected_seed_fingerprints::read_selected_seed_fingerprints; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 50b565c96b..a36b6e5b36 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -25,12 +25,6 @@ use sqlbuild_host::models::{SqlTestExecutionResult, SqlTestOutcome}; use sqlbuild_host::types::ResultEnum; use sqlbuild_orchestration::runs::models::NodeOutcome; -use crate::bindings::_helpers::build_progress::nested_progress::NestedCommandProgressCallbacks; -use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; -use crate::bindings::_helpers::build_progress::progress_text::{ - format_expectation_detail, format_expectation_name, write_execution_header, -}; -use crate::bindings::_helpers::build_progress::result_text::{test_label, test_result_label}; use crate::bindings::_helpers::change_selection::base_snapshot::{ BaseComparison, base_comparison, base_variant, }; @@ -41,10 +35,6 @@ use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_com use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::native_host::statement_report::StatementReport; -use crate::bindings::_helpers::result_output::documents::test_execution_json; -use crate::bindings::_helpers::result_output::documents::write_execution_json_output; -use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; -use crate::bindings::_helpers::result_output::resource_ids::sql_test_identity; use crate::bindings::_helpers::sql_test_command::python_tests::{ PYTHON_TESTS_INPUT_CODE, names_python_node, python_test_modules, run_python_tests, selected_python_tests, selective, with_python_section, @@ -70,6 +60,17 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_compile::compile_driver::main::setup::build_static_pipeline_result::build_static_pipeline_result; use sqlbuild_python_compile::compile_outputs::main::write_test_runtime_target::write_test_runtime_target; use sqlbuild_python_front::command_runtime::main::resolve_concurrency::resolve_concurrency; +use sqlbuild_python_output::build_progress::main::format_expectation_detail::format_expectation_detail; +use sqlbuild_python_output::build_progress::main::format_expectation_name::format_expectation_name; +use sqlbuild_python_output::build_progress::main::test_label::test_label; +use sqlbuild_python_output::build_progress::main::test_result_label::test_result_label; +use sqlbuild_python_output::build_progress::main::write_execution_header::write_execution_header; +use sqlbuild_python_output::build_progress::models::ConnectionProgressReporter; +use sqlbuild_python_output::build_progress::models::NestedCommandProgressCallbacks; +use sqlbuild_python_output::result_output::main::events::sql_test_identity::sql_test_identity; +use sqlbuild_python_output::result_output::main::results::test_execution_json::test_execution_json; +use sqlbuild_python_output::result_output::main::results::write_execution_json_output::write_execution_json_output; +use sqlbuild_python_output::result_output::models::ExecutionEventWriter; use sqlbuild_python_planner::planner_entries::main::plans::build_test_command_plan::build_test_command_plan; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; use sqlbuild_python_project::project_snapshot::types::VerifiedRun; diff --git a/fensu.toml b/fensu.toml index 5b730b140a..d4acbd188d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1467,7 +1467,7 @@ paths = ["crates/sqlbuild-python-project/src/compiled_views/_helpers/model_input reason = "Each compiled view reads its own field set from the Python boundary." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs", "crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_execution.rs"] +paths = ["crates/sqlbuild-python-output/src/build_progress/_helpers/build_callbacks.rs", "crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_execution.rs"] reason = "Each callback or worker report forwards one distinct build or check event with its own payload." [[dupes.allowlist]] @@ -1675,7 +1675,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 66768 } +thresholds = { max_crate_code_lines = 54943 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] @@ -1883,6 +1883,18 @@ path = "crates/sqlbuild-python-compile/src/input_loops/models.rs" symbols = ["LoopSwitches::extract_bound"] reason = "Python passes each of these records as one positional tuple; extract_bound reads it in a single tuple extraction and names each position, as the positional record it replaces did." +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-output/src/build_progress/models.rs" +symbols = ["NestedCommandProgressCallbacks"] +reason = "NestedCommandProgressCallbacks implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; other models derive it." + +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-output/src/recall/models.rs" +symbols = ["RecallReplacements"] +reason = "RecallReplacements implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; other models derive it." + [[targets.rust.rule_exceptions]] rule = "FPRSR307" path = "crates/sqlbuild-rules/src/scope_metadata/main.rs" @@ -2053,6 +2065,7 @@ crate_names = [ "sqlbuild-python-project", "sqlbuild-python-planner", "sqlbuild-python-compile", + "sqlbuild-python-output", "sqlbuild-python", "sqlbuild-recall", "sqlbuild-refactor", @@ -2082,11 +2095,11 @@ paths = ["crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs reason = "Each turns its own record type into its own Python objects; only the loop that collects them is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs"] +paths = ["crates/sqlbuild-python-output/src/result_output/_helpers/results/assets.rs"] reason = "Resource asset projections intentionally mirror their public JSON schemas: seeds include planning reasons, functions include warnings and their function kind, and loads include loader and skip metadata." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs"] +paths = ["crates/sqlbuild-python-output/src/result_output/_helpers/results/documents.rs"] reason = "Command document projections intentionally mirror the public envelope while preserving command-specific summaries: seeds require a plan and loads report skipped_count." [[dupes.allowlist]] @@ -2447,6 +2460,42 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/state_rows/main/event_rows/offset_aware.rs=1", "crates/sqlbuild-python-front/src/state_rows/main/migrations/old_name_retention.rs=1", "crates/sqlbuild-python-front/src/state_rows/main/migrations/read_events.rs=1", + "crates/sqlbuild-python-output/src/build_progress/_helpers/build_callbacks.rs=1", + "crates/sqlbuild-python-output/src/build_progress/_helpers/plan_facts.rs=1", + "crates/sqlbuild-python-output/src/build_progress/_helpers/progress_text.rs=3", + "crates/sqlbuild-python-output/src/build_progress/_helpers/result_text.rs=7", + "crates/sqlbuild-python-output/src/build_progress/main/test_label.rs=3", + "crates/sqlbuild-python-output/src/build_progress/main/write_execution_header.rs=2", + "crates/sqlbuild-python-output/src/build_progress/models.rs=25", + "crates/sqlbuild-python-output/src/compute_logs/_helpers/conversion.rs=7", + "crates/sqlbuild-python-output/src/compute_logs/_helpers/storage.rs=3", + "crates/sqlbuild-python-output/src/cost_telemetry/_helpers/allocation.rs=1", + "crates/sqlbuild-python-output/src/cost_telemetry/_helpers/ledger.rs=13", + "crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs=23", + "crates/sqlbuild-python-output/src/output_capture/_helpers/context.rs=5", + "crates/sqlbuild-python-output/src/output_capture/_helpers/dispatcher.rs=29", + "crates/sqlbuild-python-output/src/output_capture/_helpers/records.rs=13", + "crates/sqlbuild-python-output/src/output_capture/_helpers/values.rs=6", + "crates/sqlbuild-python-output/src/output_capture/main/invocation_context.rs=3", + "crates/sqlbuild-python-output/src/output_capture/main/register_dispatcher.rs=2", + "crates/sqlbuild-python-output/src/recall/_helpers/replacements.rs=8", + "crates/sqlbuild-python-output/src/recall/_helpers/session.rs=2", + "crates/sqlbuild-python-output/src/recall/_helpers/sinks.rs=1", + "crates/sqlbuild-python-output/src/recall/main/start_invocation_recall.rs=1", + "crates/sqlbuild-python-output/src/recall/main/write_stdout_bytes.rs=6", + "crates/sqlbuild-python-output/src/result_output/_helpers/events/event_writer.rs=4", + "crates/sqlbuild-python-output/src/result_output/_helpers/events/integration_envelope.rs=2", + "crates/sqlbuild-python-output/src/result_output/_helpers/events/integration_records.rs=3", + "crates/sqlbuild-python-output/src/result_output/_helpers/events/resource_ids.rs=1", + "crates/sqlbuild-python-output/src/result_output/_helpers/events/terminal_index.rs=3", + "crates/sqlbuild-python-output/src/result_output/_helpers/results/aggregation.rs=2", + "crates/sqlbuild-python-output/src/result_output/_helpers/results/documents.rs=4", + "crates/sqlbuild-python-output/src/result_output/_helpers/results/projection.rs=4", + "crates/sqlbuild-python-output/src/result_output/main/events/current_terminal_event_index.rs=1", + "crates/sqlbuild-python-output/src/result_output/main/events/register_terminal_index.rs=3", + "crates/sqlbuild-python-output/src/result_output/main/results/audit_execution_json.rs=3", + "crates/sqlbuild-python-output/src/result_output/main/results/write_execution_json_output.rs=6", + "crates/sqlbuild-python-output/src/result_output/models.rs=19", "crates/sqlbuild-python-planner/src/freshness/_helpers/freshness_observe.rs=6", "crates/sqlbuild-python-planner/src/freshness/_helpers/observation.rs=3", "crates/sqlbuild-python-planner/src/freshness/_helpers/planning.rs=1", @@ -2639,13 +2688,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/plan_facts.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs=13", @@ -2668,8 +2710,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_pipeline.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/conversion.rs=7", - "crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs=16", @@ -2679,10 +2719,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=23", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/translator.rs=3", @@ -2751,21 +2787,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=31", - "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/recall/writer.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs=1", @@ -2777,15 +2804,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs=23", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs=0", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs=19", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/resource_ids.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/review_command/command.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs=1", @@ -2934,6 +2952,18 @@ pending_budgets = [ "crates/sqlbuild-python-front/src/scopes/models.rs=11", "crates/sqlbuild-python-front/src/state_rows/_helpers/migrations/migration_rows.rs=1", "crates/sqlbuild-python-front/src/state_rows/models.rs=3", + "crates/sqlbuild-python-output/src/build_progress/_helpers/audit_progress.rs=2", + "crates/sqlbuild-python-output/src/build_progress/_helpers/build_callbacks.rs=2", + "crates/sqlbuild-python-output/src/build_progress/_helpers/nested_progress.rs=2", + "crates/sqlbuild-python-output/src/build_progress/models.rs=18", + "crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs=7", + "crates/sqlbuild-python-output/src/output_capture/_helpers/context.rs=2", + "crates/sqlbuild-python-output/src/output_capture/_helpers/dispatcher.rs=9", + "crates/sqlbuild-python-output/src/recall/_helpers/replacements.rs=3", + "crates/sqlbuild-python-output/src/result_output/_helpers/events/integration_envelope.rs=4", + "crates/sqlbuild-python-output/src/result_output/_helpers/events/terminal_index.rs=2", + "crates/sqlbuild-python-output/src/result_output/_helpers/results/documents.rs=8", + "crates/sqlbuild-python-output/src/result_output/models.rs=4", "crates/sqlbuild-python-planner/src/freshness/models.rs=8", "crates/sqlbuild-python-planner/src/microbatches/_helpers/conversion.rs=1", "crates/sqlbuild-python-planner/src/plan_output/_helpers/cursor_details.rs=2", @@ -2989,10 +3019,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=22", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_reporters.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=1", @@ -3010,7 +3036,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs=2", @@ -3036,18 +3061,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/review_command/comparison.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=8", @@ -3069,7 +3087,7 @@ module_layers = [ "auditing=registry", "authored_values=registry", "build_command=commands", - "build_progress=output", + "build_progress=registry", "cache=registry", "change_selection=commands", "check=commands", @@ -3086,13 +3104,13 @@ module_layers = [ "compile_outputs=registry", "compiled_project=registry", "compiled_views=registry", - "compute_logs=output", + "compute_logs=registry", "contract_command=commands", "consumption=plan", "contracts=registry", "cost_command=commands", "cost_report=output", - "cost_telemetry=output", + "cost_telemetry=registry", "cursor=registry", "dag_artifact=registry", "dagster_invocation=commands", @@ -3131,7 +3149,7 @@ module_layers = [ "native_host/resource_costs_state=front", "native_host/warehouse_metadata=commands", "observations=plan", - "output_capture=output", + "output_capture=registry", "plan_command=commands", "plan_output=registry", "planner_changes=registry", @@ -3159,7 +3177,7 @@ module_layers = [ "references=registry", "relation_naming=registry", "render=registry", - "result_output=output", + "result_output=registry", "review_command=commands", "rules=registry", "scenario_command=commands", @@ -3234,11 +3252,11 @@ paths = ["crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registrati reason = "Each bindings module registers its own functions and classes with the native module." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/event_exporting/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs"] +paths = ["crates/sqlbuild-python-base/src/event_exporting/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs", "crates/sqlbuild-python-output/src/output_capture/_helpers/context.rs", "crates/sqlbuild-python-output/src/result_output/_helpers/events/terminal_index.rs"] reason = "Each context object takes its own resource out of its own lock and releases it the way that resource needs; only the take-under-lock shape is shared." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/presentation_text/_helpers/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/getters.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/getters.rs"] +paths = ["crates/sqlbuild-python-base/src/presentation_text/_helpers/getters.rs", "crates/sqlbuild-python-output/src/build_progress/_helpers/getters.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/getters.rs"] reason = "Direct `#[getter]` readers of each module's own pyclass; each matches its own class and field names." [[dupes.allowlist]] @@ -3270,11 +3288,11 @@ paths = ["crates/sqlbuild-python-base/src/event_exporting/models.rs", "crates/sq reason = "Each context object sets its own context variable and keeps its own token under its own lock; only the set-and-store shape is shared." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/build_progress/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs"] +paths = ["crates/sqlbuild-python-base/src/build_progress/models.rs", "crates/sqlbuild-python-output/src/build_progress/_helpers/build_callbacks.rs"] reason = "The transient status line and the build progress callbacks each own their spinner thread and stop event; merging the two spinners is a separate change." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/build_progress/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs"] +paths = ["crates/sqlbuild-python-base/src/build_progress/models.rs", "crates/sqlbuild-python-output/src/build_progress/_helpers/nested_progress.rs"] reason = "The transient status line and the build progress callbacks each own their spinner thread and stop event; merging the two spinners is a separate change." [[dupes.allowlist]] @@ -3294,7 +3312,7 @@ paths = ["crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/regist reason = "Each bindings module registers its own functions and classes with the native module." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/native_host/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs"] +paths = ["crates/sqlbuild-python-front/src/native_host/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs", "crates/sqlbuild-python-output/src/result_output/_helpers/events/terminal_index.rs"] reason = "Each context object takes its own resource out of its own lock and releases it the way that resource needs; only the take-under-lock shape is shared." [[dupes.allowlist]] @@ -3446,11 +3464,11 @@ paths = ["crates/sqlbuild-python-base/src/records/_helpers/native_classes/fields reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs", "crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_validation.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_dispatch.rs"] +paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python-output/src/output_capture/_helpers/records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs", "crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_validation.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_dispatch.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_rule_evidence.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/project_tree_queries.rs"] +paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs", "crates/sqlbuild-python-planner/src/plan_output/_helpers/display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_rule_evidence.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/project_tree_queries.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3498,7 +3516,7 @@ paths = ["crates/sqlbuild-python-front/src/scopes/main/reports/register_scope_re reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs"] +paths = ["crates/sqlbuild-python-project/src/compiled_views/main/project/plain_typed_value.rs", "crates/sqlbuild-python-output/src/result_output/_helpers/results/checks.rs"] reason = "One converts a typed SQL value to plain Python values, the other to JSON-ready values; only the kind dispatch is alike." [[dupes.allowlist]] @@ -3538,7 +3556,7 @@ paths = ["crates/sqlbuild-python-project/src/analysis_session/main/shape_rows.rs reason = "Each converts its own input into its own native or Python value; only the conversion shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/compile_context/_helpers/diagnostic_collector.rs", "crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs"] +paths = ["crates/sqlbuild-python-base/src/compile_context/_helpers/diagnostic_collector.rs", "crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs", "crates/sqlbuild-python-output/src/output_capture/_helpers/context.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3666,11 +3684,11 @@ paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs", "crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_validation.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_dispatch.rs"] +paths = ["crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/deferral.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/targets.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python-output/src/output_capture/_helpers/records.rs", "crates/sqlbuild-python-compile/src/project_assembly/_helpers/syntax_validation.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/rule/rule_dispatch.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_rule_evidence.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/project_tree_queries.rs"] +paths = ["crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_rule_evidence.rs", "crates/sqlbuild-python-compile/src/rules/_helpers/checking/project_tree_queries.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3690,7 +3708,7 @@ paths = ["crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/ge reason = "Each matches its own native class's attribute names to its own fields; only the cast-and-dispatch shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/getters.rs", "crates/sqlbuild-python-compile/src/project_assembly/_helpers/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/result_output/getters.rs"] +paths = ["crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/getters.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/getters.rs", "crates/sqlbuild-python-compile/src/project_assembly/_helpers/getters.rs", "crates/sqlbuild-python-output/src/result_output/_helpers/results/getters.rs"] reason = "Each matches its own native class's attribute names to its own fields; only the cast-and-dispatch shape is alike." [[dupes.allowlist]] @@ -3770,11 +3788,11 @@ paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_targets.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_deferral.rs", "crates/sqlbuild-python-compile/src/project_assembly/main/register_syntax_validation.rs", "crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_dispatch.rs", "crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs"] +paths = ["crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_targets.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_deferral.rs", "crates/sqlbuild-python-compile/src/project_assembly/main/register_syntax_validation.rs", "crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_dispatch.rs", "crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs", "crates/sqlbuild-python-output/src/output_capture/_helpers/records.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-compile/src/rules/main/checking/register_project_tree_queries.rs", "crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_rule_evidence.rs", "crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs"] +paths = ["crates/sqlbuild-python-compile/src/rules/main/checking/register_project_tree_queries.rs", "crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_rule_evidence.rs", "crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs", "crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3849,6 +3867,78 @@ reason = "Each is one rule family's entry point: return early when no rule is se paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs"] reason = "Each builds its own dbt manifest record type with that type's fields; only the record-building shape is alike." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_targets.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/stages/register_deferral.rs", "crates/sqlbuild-python-compile/src/project_assembly/main/register_syntax_validation.rs", "crates/sqlbuild-python-compile/src/rules/main/rule/register_rule_dispatch.rs", "crates/sqlbuild-python-front/src/discovery/main/project/register_project_config.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_sql_test_records.rs", "crates/sqlbuild-python-front/src/discovery/main/records/register_yaml_records.rs", "crates/sqlbuild-python-output/src/output_capture/main/register_records.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/strategy/register_strategy.rs", "crates/sqlbuild-python-project/src/graph/main/python_nodes/register_python_selection.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/rules/main/checking/register_project_tree_queries.rs", "crates/sqlbuild-python-compile/src/rules/main/custom/register_custom_rule_evidence.rs", "crates/sqlbuild-python-front/src/contracts/main/register_effective_config.rs", "crates/sqlbuild-python-front/src/discovery/main/failures/register_guidance.rs", "crates/sqlbuild-python-front/src/model_config/main/config/register_config_values.rs", "crates/sqlbuild-python-front/src/state_rows/main/migrations/register_migration_io.rs", "crates/sqlbuild-python-output/src/cost_telemetry/main/register_snowflake.rs", "crates/sqlbuild-python-planner/src/plan_output/main/register_display.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_refs.rs", "crates/sqlbuild-python-project/src/fingerprints/main/register_storage_io.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/event_exporting/models.rs", "crates/sqlbuild-python-front/src/native_host/models.rs", "crates/sqlbuild-python-output/src/output_capture/_helpers/context.rs", "crates/sqlbuild-python-output/src/result_output/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs"] +reason = "Each context object takes its own resource out of its own lock and releases it the way that resource needs; only the take-under-lock shape is shared." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-output/src/result_output/main/results/format_load_execution_json.rs", "crates/sqlbuild-python-output/src/result_output/main/results/format_seed_execution_json.rs"] +reason = "Command document projections intentionally mirror the public envelope while preserving command-specific summaries: seeds require a plan and loads report skipped_count." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/compile_context/_helpers/diagnostic_collector.rs", "crates/sqlbuild-python-output/src/output_capture/main/register_context.rs", "crates/sqlbuild-python-project/src/planner_strategy/main/register_fixture_context.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-output/src/build_progress/main/batch_complete_callback.rs", "crates/sqlbuild-python-output/src/build_progress/main/check_callback.rs"] +reason = "Each callback or worker report forwards one distinct build or check event with its own payload." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-output/src/build_progress/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs", "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs"] +reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-output/src/build_progress/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs"] +reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-output/src/build_progress/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs"] +reason = "Each reporter toggles its own cursor state through its own writer and lock; only the hide-or-show check is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs", "crates/sqlbuild-python-output/src/result_output/_helpers/events/terminal_index.rs"] +reason = "Each reads its own context variable's installed value as its own type; only the get-or-none shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-output/src/build_progress/models.rs"] +reason = "Completion and failure update different progress counters and rows; only the locked index lookup is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_sql_bodies.rs", "crates/sqlbuild-python-output/src/build_progress/_helpers/build_callbacks.rs"] +reason = "Each reads its own record's fields from its own Python object; only the field-read shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-output/src/build_progress/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs"] +reason = "Each Python constructor reads its own keyword set into its own fields; only the keyword parsing is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/build_progress/models.rs", "crates/sqlbuild-python-output/src/build_progress/models.rs"] +reason = "Each reporter stops its own spinner and cleans up its own state; only the stop-event signal is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/build_progress/models.rs", "crates/sqlbuild-python-output/src/build_progress/models.rs"] +reason = "Each progress reporter runs its own spinner loop and draws its own line; only the wait-and-redraw loop is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-output/src/build_progress/_helpers/build_callbacks.rs", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs"] +reason = "Each reads its own record's fields from its own Python object; only the field-read shape is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-output/src/build_progress/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs"] +reason = "Each progress reporter runs its own spinner loop and draws its own line; only the wait-and-redraw loop is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-output/src/build_progress/models.rs"] +reason = "Each reporter clears its own transient line under its own lock; sharing one line writer between the reporters is a separate change." + [[dupes.allowlist]] paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs"] reason = "Each is its own Python entry point with its own keyword contract and dbt command shape; both end in the shared argv builder." diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 565dd36937..cc388cefcd 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -290,6 +290,7 @@ "crates/sqlbuild-python-project/src/", "crates/sqlbuild-python-planner/src/", "crates/sqlbuild-python-compile/src/", + "crates/sqlbuild-python-output/src/", ) NATIVE_CROSSING_PATTERNS: tuple[str, ...] = ( r"\.getattr\(", diff --git a/scripts/native_module_paths/constants.py b/scripts/native_module_paths/constants.py index cdbb904c12..6bb4658351 100644 --- a/scripts/native_module_paths/constants.py +++ b/scripts/native_module_paths/constants.py @@ -8,6 +8,7 @@ "crates/sqlbuild-python-project/src", "crates/sqlbuild-python-planner/src", "crates/sqlbuild-python-compile/src", + "crates/sqlbuild-python-output/src", "crates/sqlbuild-python/src", "crates/sqlbuild-cli/src", "crates/sqlbuild-host/src", diff --git a/tests/native_field_fallbacks.txt b/tests/native_field_fallbacks.txt index 1dd5e028ee..6ebb260f60 100644 --- a/tests/native_field_fallbacks.txt +++ b/tests/native_field_fallbacks.txt @@ -33,17 +33,15 @@ crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs Connectio crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs ConnectionProgressReporter.on_connection_error crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs ConnectionProgressReporter.on_connection_start crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs ColumnMigrationDecision.label -crates/sqlbuild-python/src/bindings/_helpers/build_progress/audit_progress.rs TransientLineCoordinator.lock -crates/sqlbuild-python/src/bindings/_helpers/build_progress/build_callbacks.rs TransientLineCoordinator.lock -crates/sqlbuild-python/src/bindings/_helpers/build_progress/nested_progress.rs TransientLineCoordinator.lock +crates/sqlbuild-python-output/src/build_progress/models.rs TransientLineCoordinator.lock crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs CliNamespace.command crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs CliNamespace.dbt_args crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs CliNamespace.json crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs CliNamespace.lineage_format crates/sqlbuild-python-base/src/build_progress/_helpers/progress.rs CliNamespace.query_format crates/sqlbuild-python-base/src/build_progress/models.rs TransientLineCoordinator.lock -crates/sqlbuild-python/src/bindings/_helpers/build_progress/progress_text.rs VerboseInspectionWriter.write -crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs SqlValueKind.value +crates/sqlbuild-python-output/src/build_progress/_helpers/progress_text.rs VerboseInspectionWriter.write +crates/sqlbuild-python-output/src/build_progress/_helpers/result_text.rs SqlValueKind.value crates/sqlbuild-python-base/src/build_progress/_helpers/terminal.rs BuildProgressCallbacks._spin_until_stopped crates/sqlbuild-python-base/src/build_progress/_helpers/terminal.rs NestedCommandProgressCallbacks._spin_until_stopped crates/sqlbuild-python-base/src/build_progress/_helpers/terminal.rs TransientStatusReporter._spin_until_stopped @@ -401,8 +399,8 @@ crates/sqlbuild-python-front/src/render/_helpers/calls/macro_call_classes.rs Dec crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_evaluation.rs NativeMacroExpansion.render_relation crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_expansion.rs ResourceKind.value crates/sqlbuild-python-front/src/render/_helpers/expansion/macro_generated_references.rs ResourceKind.value -crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs PlanReason.value -crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs SqlValueKind.value +crates/sqlbuild-python-output/src/result_output/_helpers/results/assets.rs PlanReason.value +crates/sqlbuild-python-output/src/result_output/_helpers/results/checks.rs SqlValueKind.value crates/sqlbuild-python-compile/src/rules/_helpers/custom/custom_evaluation.rs RuleSubject.value crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs ScenarioProgressRelay.on_scenario_complete crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs ScenarioProgressRelay.on_scenario_start