Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ members = [
"crates/sqlbuild-python-project",
"crates/sqlbuild-python-planner",
"crates/sqlbuild-python-compile",
"crates/sqlbuild-python-output",
"crates/sqlbuild-python",
]
resolver = "2"
Expand Down Expand Up @@ -65,9 +66,10 @@ order = [
"sqlbuild-python-project",
"sqlbuild-python-planner",
"sqlbuild-python-compile",
"sqlbuild-python-output",
"sqlbuild-python",
]
python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python-planner", "sqlbuild-python-compile", "sqlbuild-python"]
python-boundary = ["sqlbuild-python-base", "sqlbuild-python-front", "sqlbuild-python-project", "sqlbuild-python-planner", "sqlbuild-python-compile", "sqlbuild-python-output", "sqlbuild-python"]
polyglot-floor = "sqlbuild-analysis"

[workspace.package]
Expand Down Expand Up @@ -192,6 +194,7 @@ sqlbuild-python-front = { path = "crates/sqlbuild-python-front" }
sqlbuild-python-project = { path = "crates/sqlbuild-python-project" }
sqlbuild-python-planner = { path = "crates/sqlbuild-python-planner" }
sqlbuild-python-compile = { path = "crates/sqlbuild-python-compile" }
sqlbuild-python-output = { path = "crates/sqlbuild-python-output" }
sqlbuild-pysource = { path = "crates/sqlbuild-pysource" }
sqlbuild-recall = { path = "crates/sqlbuild-recall" }
sqlbuild-scopes = { path = "crates/sqlbuild-scopes" }
Expand Down
5 changes: 3 additions & 2 deletions crates/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,15 @@ tested and benchmarked with `cargo test` alone.
| `sqlbuild-python-project` | The compiled project as PyO3 bindings, on top of `sqlbuild-python-front`: the project host and the Python views of compiled models, inputs and resources, the compiled-project snapshot, analysis classes and the analysis session, the project graph, references, lineage, fingerprints, contracts and semantic shapes, audit rendering, and the planner's identities, scope, selection, phases, fixture strategy and SQL test planning. |
| `sqlbuild-python-planner` | The plan layer as PyO3 bindings, on top of `sqlbuild-python-project`: execution plans and their entries, planner phases, change detection, migrations, reference and cursor resolution, scenarios, strategy and warehouse inspection, plus freshness, consumption, microbatches, observations, lifecycle events and plan output. |
| `sqlbuild-python-compile` | The compile layer as PyO3 bindings, on top of `sqlbuild-python-project` and beside `sqlbuild-python-planner`: the compile command and driver, input loops and compile-input attachment, project assembly, analysis and the analysis session, Rules, lineage, semantic checks, compile deltas and outputs, the DAG artifact, manifest, frontier, offline scope facts, SQL test glue and auditing. |
| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-compile`, `sqlbuild-python-planner`, `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. |
| `sqlbuild-python-output` | The output layer as PyO3 bindings, on top of `sqlbuild-python-planner` and beside `sqlbuild-python-compile`: build progress reporting, build result output and its documents, output capture and invocation records, cost telemetry, compute log storage and the Recall session and its stores. |
| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-output`, `sqlbuild-python-compile`, `sqlbuild-python-planner`, `sqlbuild-python-project`, `sqlbuild-python-front` and `sqlbuild-python-base`. |

## Layering

Dependencies point one way, from the top of this graph to the bottom:

```text
sqlbuild-python -> sqlbuild-python-compile -> sqlbuild-python-planner -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base
sqlbuild-python -> sqlbuild-python-output -> sqlbuild-python-compile -> sqlbuild-python-planner -> sqlbuild-python-project -> sqlbuild-python-front -> sqlbuild-python-base
-> sqlbuild-project -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments
-> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host
-> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core
Expand Down
27 changes: 27 additions & 0 deletions crates/sqlbuild-python-output/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
[package]
name = "sqlbuild-python-output"
description = "The output layer of the sqlbuild._native bindings: build progress, result output, output capture, cost telemetry, compute logs and the Recall session."
version.workspace = true
edition.workspace = true
license.workspace = true
publish.workspace = true
rust-version.workspace = true

[dependencies]
pyo3.workspace = true
serde_json.workspace = true
sqlbuild-analysis.workspace = true
sqlbuild-cache.workspace = true
sqlbuild-cli.workspace = true
sqlbuild-config.workspace = true
sqlbuild-core.workspace = true
sqlbuild-host.workspace = true
sqlbuild-orchestration.workspace = true
sqlbuild-python-base.workspace = true
sqlbuild-python-front.workspace = true
sqlbuild-python-project.workspace = true
sqlbuild-python-planner.workspace = true
sqlbuild-recall.workspace = true

[lints]
workspace = true
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
//! Aggregate live state and plan-ordered result rows of standalone concurrent audits.
use std::collections::HashMap;

use pyo3::prelude::{Bound, Py, PyAny, PyResult};
use sqlbuild_core::nested_progress::main::audit_resource_id::audit_resource_id;

use crate::build_progress::_helpers::result_text::optional;
use crate::build_progress::_helpers::result_text::text;

/// The identity an audit plan entry or result reports to the progress projector.
pub(crate) fn resource_id(value: &Bound<'_, PyAny>, name_field: &str) -> PyResult<String> {
let kind: Option<String> = match optional(value, "attached_target_kind")? {
Some(kind) => Some(kind),
None => Some(text(value, "attachment_kind")?),
};
Ok(audit_resource_id(
&text(value, name_field)?,
kind.as_deref().unwrap_or(""),
(
optional(value, "attached_target_name")?.as_deref(),
optional(value, "attached_column_name")?.as_deref(),
),
))
}

/// Mutable reporter state; only held briefly and never across Python calls.
#[derive(Debug, Default)]
pub(crate) struct ReporterState {
pub(crate) completed_by_index: HashMap<usize, Option<Py<PyAny>>>,
pub(crate) next_index: usize,
pub(crate) running: usize,
pub(crate) completed: usize,
pub(crate) current_group: Option<String>,
pub(crate) closed: bool,
pub(crate) cursor_hidden: bool,
pub(crate) on_result: Option<Py<PyAny>>,
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
//! Live progress rows, completion rows and runtime diagnostics of `sqb build`.
use std::collections::HashMap;

use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult};
use sqlbuild_core::build_progress::models::{SchedulerStateText, StatementText, TestResultText};

use crate::build_progress::_helpers::result_text::optional;
use crate::build_progress::_helpers::result_text::text;
use sqlbuild_python_base::records::main::fields::field::field as native_field;

pub(crate) const RUNTIME_LOGGER: &str = "sqlbuild.executor.runtime";
pub(crate) const PASS_OUTCOME: &str = "pass";
pub(crate) const SCHEDULER_DIAGNOSTIC_LIMIT: usize = 50;
pub(crate) const QUERY_DIAGNOSTIC_LIMIT: usize = 25;

/// Which executor result type a completion reports.
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum ResultKind {
Model,
Seed,
Function,
Load,
PythonNode,
SqlTest,
Other,
}

pub(crate) fn result_kind(result: &Bound<'_, PyAny>) -> PyResult<ResultKind> {
for (field, kind) in [
("model_name", ResultKind::Model),
("seed_name", ResultKind::Seed),
("function_name", ResultKind::Function),
("loader_name", ResultKind::Load),
("node_name", ResultKind::PythonNode),
("test_name", ResultKind::SqlTest),
] {
if result.hasattr(field)? {
return Ok(kind);
}
}
Ok(ResultKind::Other)
}

fn sql_test_resource_id(result: &Bound<'_, PyAny>) -> PyResult<String> {
let name: String = text(result, "test_name")?;
let path: Bound<'_, PyAny> = native_field(result, "source_path")?;
let block: Bound<'_, PyAny> = native_field(result, "block_index")?;
let case: Bound<'_, PyAny> = native_field(result, "case_name")?;
if path.is_none() || block.is_none() || case.is_none() {
return Ok(format!("sql_test:{name}"));
}
Ok(format!(
"sql_test:{}:{}:{}",
path.call_method0("as_posix")?.str()?,
block.str()?,
case.str()?
))
}

/// The resource name and projector identity of a completion.
pub(crate) fn resource_identity(result: &Bound<'_, PyAny>) -> PyResult<Option<(String, String)>> {
Ok(match result_kind(result)? {
ResultKind::Model => {
let name: String = text(result, "model_name")?;
Some((name.clone(), format!("model:{name}")))
}
ResultKind::Seed => {
let name: String = text(result, "seed_name")?;
Some((name.clone(), format!("seed:{name}")))
}
ResultKind::Function => {
let name: String = text(result, "function_name")?;
let kind: String = text(result, "function_kind")?;
Some((name.clone(), format!("{kind}:{name}")))
}
ResultKind::Load => {
let name: String = text(result, "source_name")?;
Some((name.clone(), format!("source:{name}")))
}
ResultKind::PythonNode => {
let name: String = text(result, "node_name")?;
let kind: String = text(result, "kind")?;
Some((name.clone(), format!("{kind}:{name}")))
}
ResultKind::SqlTest => Some((text(result, "test_name")?, sql_test_resource_id(result)?)),
ResultKind::Other => None,
})
}

/// Mutable callback state; only held briefly and never across Python calls.
#[derive(Debug, Default)]
pub(crate) struct CallbackState {
pub(crate) counter: usize,
pub(crate) node_name: String,
pub(crate) node_kind: String,
pub(crate) sub_message: String,
pub(crate) frame: usize,
pub(crate) line_active: bool,
pub(crate) cursor_hidden: bool,
pub(crate) stop_event: Option<Py<PyAny>>,
pub(crate) thread: Option<Py<PyAny>>,
pub(crate) tests_by_model: HashMap<String, Vec<TestResultText>>,
pub(crate) scheduler_count: usize,
pub(crate) scheduler_omitted: usize,
pub(crate) last_scheduler: Option<SchedulerStateText>,
pub(crate) query_counts: HashMap<String, usize>,
pub(crate) query_omitted: usize,
pub(crate) last_query: Option<StatementText>,
}

pub(crate) fn scheduler_state(state: &Bound<'_, PyAny>) -> PyResult<SchedulerStateText> {
Ok(SchedulerStateText {
running: native_field(state, "running")?.extract()?,
ready: native_field(state, "ready")?.extract()?,
waiting: native_field(state, "waiting")?.extract()?,
limit: native_field(state, "limit")?.extract()?,
aborted: native_field(state, "aborted")?.extract()?,
})
}

pub(crate) fn statement(telemetry: &Bound<'_, PyAny>) -> PyResult<StatementText> {
Ok(StatementText {
query_id: optional(telemetry, "query_id")?,
status: text(telemetry, "status")?,
elapsed_seconds: native_field(telemetry, "elapsed_seconds")?.extract()?,
resource_type: text(telemetry, "resource_type")?,
resource_name: text(telemetry, "resource_name")?,
phase: text(telemetry, "phase")?,
})
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
//! The `sqb build` completion footer.
use crate::build_progress::main::format_build_footer_with::format_build_footer_with;
use pyo3::prelude::{Bound, PyAny, PyResult};
use pyo3::pyfunction;

use crate::build_progress::_helpers::result_text::build_text;

/// Render the build completion line, warnings, failures and skipped rollup.
#[pyfunction]
#[pyo3(signature = (*, result, elapsed, use_color, python_node_results = None))]
pub(crate) fn format_build_footer(
result: &Bound<'_, PyAny>,
elapsed: f64,
use_color: bool,
python_node_results: Option<&Bound<'_, PyAny>>,
) -> PyResult<String> {
format_build_footer_with(
&build_text(result)?,
(elapsed, use_color),
python_node_results,
Vec::new(),
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,24 +3,10 @@
use pyo3::IntoPyObjectExt;
use pyo3::prelude::{Bound, PyAny, PyResult, Python};

use crate::bindings::_helpers::build_progress::build_callbacks::BuildProgressCallbacks;
use crate::build_progress::models::BuildProgressCallbacks;
use sqlbuild_python_base::build_progress::models::TransientLineCoordinator;
use sqlbuild_python_base::records::main::registry::class_of::class_of;
use sqlbuild_python_base::records::models::GetterEntry;

/// The getter readers of this module's native pyclasses.
pub(crate) const GETTERS: &[GetterEntry] = &[
GetterEntry {
class: class_of::<BuildProgressCallbacks>,
read: read_build_progress_callbacks,
},
GetterEntry {
class: class_of::<TransientLineCoordinator>,
read: read_transient_line_coordinator,
},
];

fn read_build_progress_callbacks<'py>(
pub(crate) fn read_build_progress_callbacks<'py>(
value: &Bound<'py, PyAny>,
name: &str,
) -> Option<PyResult<Bound<'py, PyAny>>> {
Expand All @@ -37,7 +23,7 @@ fn read_build_progress_callbacks<'py>(
})
}

fn read_transient_line_coordinator<'py>(
pub(crate) fn read_transient_line_coordinator<'py>(
value: &Bound<'py, PyAny>,
name: &str,
) -> Option<PyResult<Bound<'py, PyAny>>> {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
pub(crate) mod audit_progress;
pub(crate) mod build_callbacks;
pub(crate) mod footer;
pub(crate) mod getters;
pub(crate) mod nested_progress;
pub(crate) mod plan_facts;
pub(crate) mod progress_reporters;
pub(crate) mod progress_text;
pub(crate) mod result_text;
Loading
Loading