diff --git a/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test.sql b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test.sql new file mode 100644 index 0000000000..c01ed0a0db --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test.sql @@ -0,0 +1,11 @@ +TEST (name "stg_orders__doubles_amounts"); + +WITH __ref__raw_orders AS ( + SELECT 1 AS order_id, 10 AS amount +), + +__expected__stg_orders AS ( + SELECT 1 AS order_id, 999 AS amount +) + +SELECT 1 diff --git a/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_assert.sql b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_assert.sql new file mode 100644 index 0000000000..42cf522cde --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_assert.sql @@ -0,0 +1,11 @@ +TEST (name "stg_orders__amounts_stay_small"); + +WITH __ref__raw_orders AS ( + SELECT 1 AS order_id, 10 AS amount +), + +__assert__amounts_stay_small AS ( + SELECT order_id FROM __ref("stg_orders") WHERE amount > 5 +) + +SELECT 1 diff --git a/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql new file mode 100644 index 0000000000..41cc2593cc --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql @@ -0,0 +1,11 @@ +TEST (name "stg_orders__doubles_case_amounts", parameters (amount integer), cases (ten (amount 10), twenty (amount 20))); + +WITH __ref__raw_orders AS ( + SELECT 1 AS order_id, @param("amount") AS amount +), + +__expected__stg_orders AS ( + SELECT 1 AS order_id, 999 AS amount +) + +SELECT 1 diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs index 30ef82fae9..9101403391 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs @@ -9,3 +9,5 @@ pub(crate) const STORE_BUSY_STATUS: u16 = 503; pub(crate) const JOB_SELECTOR_KINDS: [&str; 2] = ["job", "name"]; /// Separates the parts of an intersection selector. pub(crate) const SELECTOR_INTERSECTION: char = ','; +/// The key prefix of a SQL unit test's result, as the snapshot names it. +pub(crate) const SQL_TEST_KEY_PREFIX: &str = "sql_test:"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index 509d5b7530..eadd9682fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -1,6 +1,6 @@ //! Frozen selection, executor callbacks and completion belong to one verified worker build. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; @@ -9,14 +9,21 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyStringMethods; use sqlbuild_core::models::ResourceKind; use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; +use sqlbuild_host::models::SqlTestOutcome; +use sqlbuild_host::types::ResultEnum; use sqlbuild_orchestration::runs::models::{ NodeOutcome, RunChange, RunCommand, RunExecution, RunOutcome, }; -use crate::bindings::_helpers::compiled_project::constants::{NO_CHANGE_REASON, SKIP_ACTION}; +use crate::bindings::_helpers::compiled_project::constants::{ + NO_CHANGE_REASON, SKIP_ACTION, SQL_TEST_KEY_PREFIX, +}; use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; use crate::bindings::_helpers::records::fields::field; +/// Node keys by snapshot node key, such as the models each test checks. +pub(crate) type NodeKeys = BTreeMap>; + /// This transient owner records only acknowledged outcomes; durable lifecycle remains on the API. #[derive(Debug)] pub(crate) struct WorkerBuild { @@ -26,6 +33,12 @@ pub(crate) struct WorkerBuild { checks: Mutex>>, /// The snapshot's audits and SQL tests, each with the node keys it checks. attached: BTreeMap>, + /// Each SQL test with every model a failure of it blocks in a build: its expected, assertion and reference targets. + blocked: BTreeMap>, + /// Each snapshot node's direct upstream node keys, for naming what a skipped node waited on. + upstream: BTreeMap>, + /// The failed node each finished node traces back to: itself when it failed, its cause when skipped. + causes: Mutex>, /// The worst attached outcome of each node, for commands that only run tests or audits. checked: Mutex>, /// Whether the command returned, having run every check it planned, rather than being refused or raising. @@ -37,7 +50,7 @@ pub(crate) struct WorkerBuild { impl WorkerBuild { pub(crate) fn new( worker: &Bound<'_, NativeOrchestrationWorker>, - attached: BTreeMap>, + (attached, blocked, upstream): (NodeKeys, NodeKeys, NodeKeys), ) -> PyResult> { let state = Arc::new(Self { execution: worker.get().execution(worker.py())?, @@ -45,6 +58,9 @@ impl WorkerBuild { completed: Mutex::default(), checks: Mutex::default(), attached, + blocked, + upstream, + causes: Mutex::default(), checked: Mutex::default(), returned: AtomicBool::new(false), refusal: Mutex::default(), @@ -174,6 +190,7 @@ impl WorkerBuild { Ok(()) } + /// A skip in a build names the failed node upstream of it; a test or audit run that raised skips what it never checked, with no cause. fn complete( &self, py: Python<'_>, @@ -195,6 +212,14 @@ impl WorkerBuild { { return Ok(()); } + let skipped_because_of = match outcome { + NodeOutcome::Skipped => match self.execution.skipped_nodes.get(key) { + Some(cause) => Some(cause.clone()), + None if !self.checks_only() => self.failed_upstream(key)?, + None => None, + }, + _ => None, + }; self.worker.bind(py).get().change( py, &format!("node-finished/{key}"), @@ -203,10 +228,20 @@ impl WorkerBuild { outcome, rows_affected, error_code, - skipped_because_of: self.execution.skipped_nodes.get(key).cloned(), + skipped_because_of: skipped_because_of.clone(), error_message: error_message.as_deref().map(redact_error_text), }, )?; + let cause = match outcome { + NodeOutcome::Failed => Some(key.to_owned()), + _ => skipped_because_of, + }; + if let Some(cause) = cause { + self.causes + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), cause); + } self.completed .lock() .map_err(|_| poisoned())? @@ -331,8 +366,58 @@ impl WorkerBuild { Ok(()) } + /// The failed node nearest upstream of `key` in this run, or the cause a skipped one carries. + fn failed_upstream(&self, key: &str) -> PyResult> { + let causes = self.causes.lock().map_err(|_| poisoned())?; + let mut seen: BTreeSet<&str> = BTreeSet::from([key]); + let mut pending: VecDeque<&str> = VecDeque::from([key]); + while let Some(node) = pending.pop_front() { + for upstream in self.upstream.get(node).into_iter().flatten() { + if let Some(cause) = causes.get(upstream) { + return Ok(Some(cause.clone())); + } + if seen.insert(upstream) { + pending.push_back(upstream); + } + } + } + Ok(None) + } + + /// A failing SQL test, found by its compiled name such as `parent [case]`, fails the models it blocks with its error. + fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { + let py = result.py(); + let outcome: String = field(result, "outcome")?.str()?.extract()?; + if SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass) + || self.worker.bind(py).get().refresh_cancellation(py)? + { + return Ok(()); + } + let name: String = field(result, "test_name")?.extract()?; + let key = format!("{SQL_TEST_KEY_PREFIX}{name}"); + let error: Option = match result.getattr_opt("error_code")? { + Some(value) => value.extract()?, + None => None, + }; + let message: Option = match result.getattr_opt("error_message")? { + Some(value) => value.extract()?, + None => None, + }; + for model in self.blocked.get(&key).into_iter().flatten() { + self.complete( + py, + model, + (NodeOutcome::Failed, None, (error.clone(), message.clone())), + )?; + } + Ok(()) + } + /// Existing native progress callbacks publish selected-node evidence without Python relays. pub(crate) fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { + if key.starts_with(SQL_TEST_KEY_PREFIX) { + return self.finished_test(result); + } if self.execution.claim.request.selection.contains(key) { let status: String = field(result, "status")?.str()?.extract()?; let outcome = match status.as_str() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 92cc25b235..1d6c1c9bee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -19,13 +19,14 @@ use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::pipeline::compiled_phase; use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; +use crate::bindings::_helpers::compiled_project::snapshot_execution::NodeKeys; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::location_command::constants::{EXECUTE_ACTION, PUBLISH_ACTION}; use crate::bindings::_helpers::location_command::runtime::run_runtime; use crate::bindings::_helpers::location_command::snapshot::{ build_orchestration_snapshot, orchestration_snapshot, }; -use crate::bindings::_helpers::location_command::worker::execute_worker; +use crate::bindings::_helpers::location_command::worker::{blocked_nodes, execute_worker}; use crate::bindings::_helpers::native_host::host::write_console; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, @@ -113,7 +114,7 @@ fn compile_location<'py>( py: Python<'py>, arguments: &KeywordArguments<'py>, input: &SnapshotInput, -) -> PyResult<(NativeOrchestrationSnapshot, Bound<'py, PyAny>)> { +) -> PyResult<(NativeOrchestrationSnapshot, Bound<'py, PyAny>, NodeKeys)> { status(py, "Discovering location project...")?; let ProjectAdapter { project_dir, @@ -152,7 +153,7 @@ fn compile_location<'py>( ), )?; status(py, "Location snapshot compiled.")?; - Ok((snapshot, discovered)) + Ok((snapshot, discovered, blocked_nodes(&project)?)) } pub(super) fn location_arguments<'py>( @@ -205,7 +206,7 @@ pub(super) fn publish_location( .value("project_dir")? .extract::()?; let input = metadata(py, arguments, &directory)?; - let (snapshot, _) = compile_location(py, arguments, &input)?; + let (snapshot, _, _) = compile_location(py, arguments, &input)?; let locations: BTreeSet<_> = snapshot .input .nodes @@ -287,10 +288,10 @@ pub(crate) fn run_location<'py>( arguments.value("run_id")?.extract::()?, arguments.value("claim_id")?.extract::()?, ); - let (snapshot, discovered) = compile_location(py, &arguments, &input)?; + let (snapshot, discovered, blocked) = compile_location(py, &arguments, &input)?; return execute_worker( py, - (&snapshot, discovered), + (&snapshot, discovered, blocked), &arguments, (&bearer, identity, recall), ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 4d28d969ae..00681e4458 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -26,9 +26,11 @@ use crate::bindings::_helpers::command_runtime::warehouses::{ WarehouseInputs, configured_warehouses, }; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; +use crate::bindings::_helpers::compiled_project::constants::SQL_TEST_KEY_PREFIX; use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; -use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; +use crate::bindings::_helpers::compiled_project::snapshot_execution::{NodeKeys, WorkerBuild}; use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; +use crate::bindings::_helpers::compiled_views::project_reads::sql_tests; use crate::bindings::_helpers::discovery::project_config::configured_changes; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::constants::{ @@ -73,6 +75,38 @@ fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap) -> PyResult { + let mut blocked: NodeKeys = BTreeMap::new(); + for test in sql_tests(project)?.try_iter()? { + let test = test?; + let name: String = field(&field(&test, "key")?, "name")?.extract()?; + let models: Vec = field(&test, "target_model_names")?.extract()?; + blocked.insert( + format!("{SQL_TEST_KEY_PREFIX}{name}"), + models + .into_iter() + .map(|model| format!("model:{model}")) + .collect(), + ); + } + Ok(blocked) +} + +/// Each snapshot node's direct upstream node keys. +fn upstream_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap> { + snapshot + .input + .nodes + .iter() + .map(|(key, node)| { + let deps = node.definition.get("deps").and_then(Value::as_array); + let deps = deps.into_iter().flatten().filter_map(Value::as_str); + (key.clone(), deps.map(str::to_owned).collect()) + }) + .collect() +} + /// A job run's `--changed` settings: its whole selection, and source data counted unless configured. fn job_run_settings( project_dir: &Bound<'_, PyAny>, @@ -257,7 +291,7 @@ fn run_cancellable<'py>( pub(super) fn execute_worker<'py>( py: Python<'py>, - (snapshot, discovered): (&NativeOrchestrationSnapshot, Bound<'py, PyAny>), + (snapshot, discovered, blocked): (&NativeOrchestrationSnapshot, Bound<'py, PyAny>, NodeKeys), arguments: &KeywordArguments<'py>, (bearer, identity, recall): (&str, (String, String), Option<&InvocationRecall>), ) -> PyResult { @@ -272,7 +306,10 @@ pub(super) fn execute_worker<'py>( ) .map_err(|error| PyValueError::new_err(error.to_string()))?, )?; - let worker = WorkerBuild::new(&owner, attached_nodes(snapshot))?; + let worker = WorkerBuild::new( + &owner, + (attached_nodes(snapshot), blocked, upstream_nodes(snapshot)), + )?; if let Some(recall) = recall { *recall .worker diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index b230cebb88..fc4f1f5988 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -123,6 +123,10 @@ class WorkerExecutionTestCase: concurrency_limits: dict[str, int] = field(default_factory=dict) launcher: Literal["native", "uncooperative", "docker"] = "native" warehouse_setup: Callable[[Path], None] = lambda _: None + # Each run node as (key, status, error code, skipped because of, error message). + expected_run_nodes: list[tuple[str, str, str | None, str | None, str | None]] = field( + default_factory=list + ) @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index ff3cfb341e..2c286f5d48 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -198,6 +198,166 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "a failing unit test fails its model, and what reads it is skipped because of it", + "SELECT 3 AS amount", + "SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('stg_orders', 'order_totals')", + "partial", + [(0,)], + files={ + "models/raw_orders.sql": ( + "MODEL (description 'Raw orders.', materialized table);\n" + "SELECT 1 AS order_id, 10 AS amount\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\n" + 'SELECT order_id, amount * 2 AS amount FROM __ref("raw_orders")\n' + ), + "models/order_totals.sql": ( + "MODEL (description 'Order totals.', materialized table);\n" + 'SELECT SUM(amount) AS total FROM __ref("stg_orders")\n' + ), + "tests/unit/test_stg_orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test.sql" + ).read_text(), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + 'annual_orders = job("annual_orders", ' + 'select="orders raw_orders stg_orders order_totals", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + }, + expected_run_nodes=[ + ("model:order_totals", "skipped", None, "model:stg_orders", None), + ("model:orders", "success", None, None, None), + ("model:raw_orders", "success", None, None, None), + ( + "model:stg_orders", + "failed", + "T003", + None, + "test 'stg_orders__doubles_amounts' failed for models: stg_orders", + ), + ], + ), + WorkerExecutionTestCase( + "a failing case of a parameterised test fails its model", + "SELECT 3 AS amount", + "SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('stg_orders', 'order_totals')", + "partial", + [(0,)], + files={ + "models/raw_orders.sql": ( + "MODEL (description 'Raw orders.', materialized table);\n" + "SELECT 1 AS order_id, 10 AS amount\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\n" + 'SELECT order_id, amount * 2 AS amount FROM __ref("raw_orders")\n' + ), + "models/order_totals.sql": ( + "MODEL (description 'Order totals.', materialized table);\n" + 'SELECT SUM(amount) AS total FROM __ref("stg_orders")\n' + ), + "tests/unit/test_stg_orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql" + ).read_text(), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + 'annual_orders = job("annual_orders", ' + 'select="orders raw_orders stg_orders order_totals", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + }, + expected_run_nodes=[ + ("model:order_totals", "skipped", None, "model:stg_orders", None), + ("model:orders", "success", None, None, None), + ("model:raw_orders", "success", None, None, None), + ( + "model:stg_orders", + "failed", + "T003", + None, + "test 'stg_orders__doubles_case_amounts [ten]' failed for models: stg_orders", + ), + ], + ), + WorkerExecutionTestCase( + "a failing assertion fails the model it reads, with no expected output", + "SELECT 3 AS amount", + "SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('stg_orders', 'order_totals')", + "partial", + [(0,)], + files={ + "models/raw_orders.sql": ( + "MODEL (description 'Raw orders.', materialized table);\n" + "SELECT 1 AS order_id, 10 AS amount\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\n" + 'SELECT order_id, amount * 2 AS amount FROM __ref("raw_orders")\n' + ), + "models/order_totals.sql": ( + "MODEL (description 'Order totals.', materialized table);\n" + 'SELECT SUM(amount) AS total FROM __ref("stg_orders")\n' + ), + "tests/unit/test_stg_orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_assert.sql" + ).read_text(), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + 'annual_orders = job("annual_orders", ' + 'select="orders raw_orders stg_orders order_totals", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + }, + expected_run_nodes=[ + ("model:order_totals", "skipped", None, "model:stg_orders", None), + ("model:orders", "success", None, None, None), + ("model:raw_orders", "success", None, None, None), + ( + "model:stg_orders", + "failed", + "T003", + None, + "test 'stg_orders__amounts_stay_small' failed for models: assertion amounts_stay_small", + ), + ], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_failing_unit_test_when_a_worker_builds_then_its_model_fails_and_downstream_names_it( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert run["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + nodes: list[tuple[str, str, str | None, str | None, str | None]] = control.execute( + "SELECT n.node_key, n.status, n.error_code, n.skipped_because_of, " + "json_extract(e.payload, '$.change.change.error_message') " + "FROM sqb_run_nodes n LEFT JOIN recall_events e " + "ON e.storage_id=n.finished_storage_id " + "WHERE n.run_id=? AND n.kind='node' ORDER BY n.node_key", + (run["run_id"],), + ).fetchall() + assert nodes == test_case.expected_run_nodes, worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + @pytest.mark.parametrize( "test_case", [