From 944abb42ae54d9bf3b15b060d6b2d4457c9cd0a3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:42:00 +0000 Subject: [PATCH 1/3] fix(worker): record a model blocked by a failing unit test as failed, and name the failed node on downstream skips --- .../orchestration/worker_failed_unit_test.sql | 11 +++ .../_helpers/compiled_project/constants.rs | 4 + .../compiled_project/snapshot_execution.rs | 79 ++++++++++++++++++- .../_helpers/location_command/worker.rs | 16 +++- .../publication/test_worker.py | 71 +++++++++++++++++ 5 files changed, 176 insertions(+), 5 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test.sql diff --git a/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test.sql b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test.sql new file mode 100644 index 0000000000..c01ed0a0db --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test.sql @@ -0,0 +1,11 @@ +TEST (name "stg_orders__doubles_amounts"); + +WITH __ref__raw_orders AS ( + SELECT 1 AS order_id, 10 AS amount +), + +__expected__stg_orders AS ( + SELECT 1 AS order_id, 999 AS amount +) + +SELECT 1 diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs index 30ef82fae9..5e893a804f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs @@ -9,3 +9,7 @@ pub(crate) const STORE_BUSY_STATUS: u16 = 503; pub(crate) const JOB_SELECTOR_KINDS: [&str; 2] = ["job", "name"]; /// Separates the parts of an intersection selector. pub(crate) const SELECTOR_INTERSECTION: char = ','; +/// The key prefix of a SQL unit test's result, as the snapshot names it. +pub(crate) const SQL_TEST_KEY_PREFIX: &str = "sql_test:"; +/// The outcome of a SQL unit test that passed. +pub(crate) const SQL_TEST_PASS: &str = "pass"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index 509d5b7530..18fc0aa519 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -1,6 +1,6 @@ //! Frozen selection, executor callbacks and completion belong to one verified worker build. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; @@ -13,7 +13,9 @@ use sqlbuild_orchestration::runs::models::{ NodeOutcome, RunChange, RunCommand, RunExecution, RunOutcome, }; -use crate::bindings::_helpers::compiled_project::constants::{NO_CHANGE_REASON, SKIP_ACTION}; +use crate::bindings::_helpers::compiled_project::constants::{ + NO_CHANGE_REASON, SKIP_ACTION, SQL_TEST_KEY_PREFIX, SQL_TEST_PASS, +}; use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; use crate::bindings::_helpers::records::fields::field; @@ -26,6 +28,10 @@ pub(crate) struct WorkerBuild { checks: Mutex>>, /// The snapshot's audits and SQL tests, each with the node keys it checks. attached: BTreeMap>, + /// Each snapshot node's direct upstream node keys, for naming what a skipped node waited on. + upstream: BTreeMap>, + /// The failed node each finished node traces back to: itself when it failed, its cause when skipped. + causes: Mutex>, /// The worst attached outcome of each node, for commands that only run tests or audits. checked: Mutex>, /// Whether the command returned, having run every check it planned, rather than being refused or raising. @@ -37,7 +43,7 @@ pub(crate) struct WorkerBuild { impl WorkerBuild { pub(crate) fn new( worker: &Bound<'_, NativeOrchestrationWorker>, - attached: BTreeMap>, + (attached, upstream): (BTreeMap>, BTreeMap>), ) -> PyResult> { let state = Arc::new(Self { execution: worker.get().execution(worker.py())?, @@ -45,6 +51,8 @@ impl WorkerBuild { completed: Mutex::default(), checks: Mutex::default(), attached, + upstream, + causes: Mutex::default(), checked: Mutex::default(), returned: AtomicBool::new(false), refusal: Mutex::default(), @@ -195,6 +203,13 @@ impl WorkerBuild { { return Ok(()); } + let skipped_because_of = match outcome { + NodeOutcome::Skipped => match self.execution.skipped_nodes.get(key) { + Some(cause) => Some(cause.clone()), + None => self.failed_upstream(key)?, + }, + _ => None, + }; self.worker.bind(py).get().change( py, &format!("node-finished/{key}"), @@ -203,10 +218,20 @@ impl WorkerBuild { outcome, rows_affected, error_code, - skipped_because_of: self.execution.skipped_nodes.get(key).cloned(), + skipped_because_of: skipped_because_of.clone(), error_message: error_message.as_deref().map(redact_error_text), }, )?; + let cause = match outcome { + NodeOutcome::Failed => Some(key.to_owned()), + _ => skipped_because_of, + }; + if let Some(cause) = cause { + self.causes + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), cause); + } self.completed .lock() .map_err(|_| poisoned())? @@ -331,8 +356,54 @@ impl WorkerBuild { Ok(()) } + /// The failed node nearest upstream of `key` in this run, or the cause a skipped one carries. + fn failed_upstream(&self, key: &str) -> PyResult> { + let causes = self.causes.lock().map_err(|_| poisoned())?; + let mut seen: BTreeSet<&str> = BTreeSet::from([key]); + let mut pending: VecDeque<&str> = VecDeque::from([key]); + while let Some(node) = pending.pop_front() { + for upstream in self.upstream.get(node).into_iter().flatten() { + if let Some(cause) = causes.get(upstream) { + return Ok(Some(cause.clone())); + } + if seen.insert(upstream) { + pending.push_back(upstream); + } + } + } + Ok(None) + } + + /// A failing SQL test blocks the models it covers, so they finish failed with its error rather than as bare skips. + fn finished_test(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { + let py = result.py(); + let outcome: String = field(result, "outcome")?.str()?.extract()?; + if outcome == SQL_TEST_PASS || self.worker.bind(py).get().refresh_cancellation(py)? { + return Ok(()); + } + let error: Option = match result.getattr_opt("error_code")? { + Some(value) => value.extract()?, + None => None, + }; + let message: Option = match result.getattr_opt("error_message")? { + Some(value) => value.extract()?, + None => None, + }; + for model in self.attached.get(key).into_iter().flatten() { + self.complete( + py, + model, + (NodeOutcome::Failed, None, (error.clone(), message.clone())), + )?; + } + Ok(()) + } + /// Existing native progress callbacks publish selected-node evidence without Python relays. pub(crate) fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { + if key.starts_with(SQL_TEST_KEY_PREFIX) { + return self.finished_test(result, key); + } if self.execution.claim.request.selection.contains(key) { let status: String = field(result, "status")?.str()?.extract()?; let outcome = match status.as_str() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 4d28d969ae..c1bcb6653f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -73,6 +73,20 @@ fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap BTreeMap> { + snapshot + .input + .nodes + .iter() + .map(|(key, node)| { + let deps = node.definition.get("deps").and_then(Value::as_array); + let deps = deps.into_iter().flatten().filter_map(Value::as_str); + (key.clone(), deps.map(str::to_owned).collect()) + }) + .collect() +} + /// A job run's `--changed` settings: its whole selection, and source data counted unless configured. fn job_run_settings( project_dir: &Bound<'_, PyAny>, @@ -272,7 +286,7 @@ pub(super) fn execute_worker<'py>( ) .map_err(|error| PyValueError::new_err(error.to_string()))?, )?; - let worker = WorkerBuild::new(&owner, attached_nodes(snapshot))?; + let worker = WorkerBuild::new(&owner, (attached_nodes(snapshot), upstream_nodes(snapshot)))?; if let Some(recall) = recall { *recall .worker diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index ff3cfb341e..7494f5ed13 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -198,6 +198,77 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "a failing unit test fails its model, and what reads it is skipped because of it", + "SELECT 3 AS amount", + "SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('stg_orders', 'order_totals')", + "partial", + [(0,)], + files={ + "models/raw_orders.sql": ( + "MODEL (description 'Raw orders.', materialized table);\n" + "SELECT 1 AS order_id, 10 AS amount\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\n" + 'SELECT order_id, amount * 2 AS amount FROM __ref("raw_orders")\n' + ), + "models/order_totals.sql": ( + "MODEL (description 'Order totals.', materialized table);\n" + 'SELECT SUM(amount) AS total FROM __ref("stg_orders")\n' + ), + "tests/unit/test_stg_orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test.sql" + ).read_text(), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + 'annual_orders = job("annual_orders", ' + 'select="orders raw_orders stg_orders order_totals", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_failing_unit_test_when_a_worker_builds_then_its_model_fails_and_downstream_names_it( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert run["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + nodes: list[tuple[str, str, str | None, str | None, str | None]] = control.execute( + "SELECT n.node_key, n.status, n.error_code, n.skipped_because_of, " + "json_extract(e.payload, '$.change.change.error_message') " + "FROM sqb_run_nodes n LEFT JOIN recall_events e " + "ON e.storage_id=n.finished_storage_id " + "WHERE n.run_id=? AND n.kind='node' ORDER BY n.node_key", + (run["run_id"],), + ).fetchall() + assert nodes == [ + ("model:order_totals", "skipped", None, "model:stg_orders", None), + ("model:orders", "success", None, None, None), + ("model:raw_orders", "success", None, None, None), + ( + "model:stg_orders", + "failed", + "T003", + None, + "test 'stg_orders__doubles_amounts' failed for models: stg_orders", + ), + ], worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + @pytest.mark.parametrize( "test_case", [ From 86368157fafc7837e5668fba8bce535e657785ce Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:02:46 +0000 Subject: [PATCH 2/3] fix(worker): block parameterised and assertion-only test targets, and keep check runs' skips uncaused --- .../worker_failed_unit_test_assert.sql | 11 ++ .../worker_failed_unit_test_cases.sql | 11 ++ .../_helpers/compiled_project/constants.rs | 2 - .../compiled_project/snapshot_execution.rs | 31 +++-- .../_helpers/location_command/command.rs | 13 +- .../_helpers/location_command/constants.rs | 6 + .../_helpers/location_command/worker.rs | 37 +++++- .../publication/_test_types.py | 4 + .../publication/test_worker.py | 115 ++++++++++++++++-- 9 files changed, 197 insertions(+), 33 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_assert.sql create mode 100644 benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql diff --git a/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_assert.sql b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_assert.sql new file mode 100644 index 0000000000..42cf522cde --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_assert.sql @@ -0,0 +1,11 @@ +TEST (name "stg_orders__amounts_stay_small"); + +WITH __ref__raw_orders AS ( + SELECT 1 AS order_id, 10 AS amount +), + +__assert__amounts_stay_small AS ( + SELECT order_id FROM __ref("stg_orders") WHERE amount > 5 +) + +SELECT 1 diff --git a/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql new file mode 100644 index 0000000000..41cc2593cc --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql @@ -0,0 +1,11 @@ +TEST (name "stg_orders__doubles_case_amounts", parameters (amount integer), cases (ten (amount 10), twenty (amount 20))); + +WITH __ref__raw_orders AS ( + SELECT 1 AS order_id, @param("amount") AS amount +), + +__expected__stg_orders AS ( + SELECT 1 AS order_id, 999 AS amount +) + +SELECT 1 diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs index 5e893a804f..9101403391 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs @@ -11,5 +11,3 @@ pub(crate) const JOB_SELECTOR_KINDS: [&str; 2] = ["job", "name"]; pub(crate) const SELECTOR_INTERSECTION: char = ','; /// The key prefix of a SQL unit test's result, as the snapshot names it. pub(crate) const SQL_TEST_KEY_PREFIX: &str = "sql_test:"; -/// The outcome of a SQL unit test that passed. -pub(crate) const SQL_TEST_PASS: &str = "pass"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index 18fc0aa519..03abfdacc5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -9,16 +9,21 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyStringMethods; use sqlbuild_core::models::ResourceKind; use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; +use sqlbuild_host::models::SqlTestOutcome; +use sqlbuild_host::types::ResultEnum; use sqlbuild_orchestration::runs::models::{ NodeOutcome, RunChange, RunCommand, RunExecution, RunOutcome, }; use crate::bindings::_helpers::compiled_project::constants::{ - NO_CHANGE_REASON, SKIP_ACTION, SQL_TEST_KEY_PREFIX, SQL_TEST_PASS, + NO_CHANGE_REASON, SKIP_ACTION, SQL_TEST_KEY_PREFIX, }; use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; use crate::bindings::_helpers::records::fields::field; +/// Node keys by snapshot node key, such as the models each test checks. +pub(crate) type NodeKeys = BTreeMap>; + /// This transient owner records only acknowledged outcomes; durable lifecycle remains on the API. #[derive(Debug)] pub(crate) struct WorkerBuild { @@ -28,6 +33,8 @@ pub(crate) struct WorkerBuild { checks: Mutex>>, /// The snapshot's audits and SQL tests, each with the node keys it checks. attached: BTreeMap>, + /// Each SQL test with every model a failure of it blocks in a build: its expected, assertion and reference targets. + blocked: BTreeMap>, /// Each snapshot node's direct upstream node keys, for naming what a skipped node waited on. upstream: BTreeMap>, /// The failed node each finished node traces back to: itself when it failed, its cause when skipped. @@ -43,7 +50,7 @@ pub(crate) struct WorkerBuild { impl WorkerBuild { pub(crate) fn new( worker: &Bound<'_, NativeOrchestrationWorker>, - (attached, upstream): (BTreeMap>, BTreeMap>), + (attached, blocked, upstream): (NodeKeys, NodeKeys, NodeKeys), ) -> PyResult> { let state = Arc::new(Self { execution: worker.get().execution(worker.py())?, @@ -51,6 +58,7 @@ impl WorkerBuild { completed: Mutex::default(), checks: Mutex::default(), attached, + blocked, upstream, causes: Mutex::default(), checked: Mutex::default(), @@ -203,10 +211,12 @@ impl WorkerBuild { { return Ok(()); } + // A test or audit run that raised skips what it never checked, not what a failed check reads. let skipped_because_of = match outcome { NodeOutcome::Skipped => match self.execution.skipped_nodes.get(key) { Some(cause) => Some(cause.clone()), - None => self.failed_upstream(key)?, + None if !self.checks_only() => self.failed_upstream(key)?, + None => None, }, _ => None, }; @@ -375,12 +385,17 @@ impl WorkerBuild { } /// A failing SQL test blocks the models it covers, so they finish failed with its error rather than as bare skips. - fn finished_test(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { + fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { let py = result.py(); - let outcome: String = field(result, "outcome")?.str()?.extract()?; - if outcome == SQL_TEST_PASS || self.worker.bind(py).get().refresh_cancellation(py)? { + let outcome: String = field(result, "outcome")?.getattr("value")?.extract()?; + if SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass) + || self.worker.bind(py).get().refresh_cancellation(py)? + { return Ok(()); } + // The snapshot keys a parameterised case by its compiled name, not by the file and block the result reports. + let name: String = field(result, "test_name")?.extract()?; + let key = format!("{SQL_TEST_KEY_PREFIX}{name}"); let error: Option = match result.getattr_opt("error_code")? { Some(value) => value.extract()?, None => None, @@ -389,7 +404,7 @@ impl WorkerBuild { Some(value) => value.extract()?, None => None, }; - for model in self.attached.get(key).into_iter().flatten() { + for model in self.blocked.get(&key).into_iter().flatten() { self.complete( py, model, @@ -402,7 +417,7 @@ impl WorkerBuild { /// Existing native progress callbacks publish selected-node evidence without Python relays. pub(crate) fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { if key.starts_with(SQL_TEST_KEY_PREFIX) { - return self.finished_test(result, key); + return self.finished_test(result); } if self.execution.claim.request.selection.contains(key) { let status: String = field(result, "status")?.str()?.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 92cc25b235..1d6c1c9bee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -19,13 +19,14 @@ use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::pipeline::compiled_phase; use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; +use crate::bindings::_helpers::compiled_project::snapshot_execution::NodeKeys; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::location_command::constants::{EXECUTE_ACTION, PUBLISH_ACTION}; use crate::bindings::_helpers::location_command::runtime::run_runtime; use crate::bindings::_helpers::location_command::snapshot::{ build_orchestration_snapshot, orchestration_snapshot, }; -use crate::bindings::_helpers::location_command::worker::execute_worker; +use crate::bindings::_helpers::location_command::worker::{blocked_nodes, execute_worker}; use crate::bindings::_helpers::native_host::host::write_console; use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, @@ -113,7 +114,7 @@ fn compile_location<'py>( py: Python<'py>, arguments: &KeywordArguments<'py>, input: &SnapshotInput, -) -> PyResult<(NativeOrchestrationSnapshot, Bound<'py, PyAny>)> { +) -> PyResult<(NativeOrchestrationSnapshot, Bound<'py, PyAny>, NodeKeys)> { status(py, "Discovering location project...")?; let ProjectAdapter { project_dir, @@ -152,7 +153,7 @@ fn compile_location<'py>( ), )?; status(py, "Location snapshot compiled.")?; - Ok((snapshot, discovered)) + Ok((snapshot, discovered, blocked_nodes(&project)?)) } pub(super) fn location_arguments<'py>( @@ -205,7 +206,7 @@ pub(super) fn publish_location( .value("project_dir")? .extract::()?; let input = metadata(py, arguments, &directory)?; - let (snapshot, _) = compile_location(py, arguments, &input)?; + let (snapshot, _, _) = compile_location(py, arguments, &input)?; let locations: BTreeSet<_> = snapshot .input .nodes @@ -287,10 +288,10 @@ pub(crate) fn run_location<'py>( arguments.value("run_id")?.extract::()?, arguments.value("claim_id")?.extract::()?, ); - let (snapshot, discovered) = compile_location(py, &arguments, &input)?; + let (snapshot, discovered, blocked) = compile_location(py, &arguments, &input)?; return execute_worker( py, - (&snapshot, discovered), + (&snapshot, discovered, blocked), &arguments, (&bearer, identity, recall), ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index b9e42eaa95..3095bcb0ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -33,3 +33,9 @@ pub(crate) const REACH_SETTING: &str = "reach"; pub(crate) const JOB_RUN_REACH: &str = "all"; /// What a job run with `changed: True` counts when `[changes] counts` is not set: every change. pub(crate) const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; +/// The model lists of a compiled SQL test that a failure of it blocks in a build. +pub(crate) const BLOCKED_TEST_TARGETS: [&str; 3] = [ + "expected_model_names", + "assertion_target_model_names", + "reference_target_model_names", +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index c1bcb6653f..e1b76ad979 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -26,13 +26,16 @@ use crate::bindings::_helpers::command_runtime::warehouses::{ WarehouseInputs, configured_warehouses, }; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; +use crate::bindings::_helpers::compiled_project::constants::SQL_TEST_KEY_PREFIX; use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; -use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; +use crate::bindings::_helpers::compiled_project::snapshot_execution::{NodeKeys, WorkerBuild}; use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker; +use crate::bindings::_helpers::compiled_views::project_reads::sql_tests; use crate::bindings::_helpers::discovery::project_config::configured_changes; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::constants::{ - CHANGED_SETTING, COMMAND_IMPLEMENTATIONS, JOB_RUN_COUNTS, JOB_RUN_REACH, REACH_SETTING, + BLOCKED_TEST_TARGETS, CHANGED_SETTING, COMMAND_IMPLEMENTATIONS, JOB_RUN_COUNTS, JOB_RUN_REACH, + REACH_SETTING, }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::recall::console::write_stderr; @@ -73,6 +76,29 @@ fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap) -> PyResult { + let mut blocked: NodeKeys = BTreeMap::new(); + for test in sql_tests(project)?.try_iter()? { + let test = test?; + let name: String = field(&field(&test, "key")?, "name")?.extract()?; + let mut models: Vec = Vec::new(); + for list in BLOCKED_TEST_TARGETS { + let Some(names) = test.getattr_opt(list)? else { + continue; + }; + for name in names.extract::>()? { + let model = format!("model:{name}"); + if !models.contains(&model) { + models.push(model); + } + } + } + blocked.insert(format!("{SQL_TEST_KEY_PREFIX}{name}"), models); + } + Ok(blocked) +} + /// Each snapshot node's direct upstream node keys. fn upstream_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap> { snapshot @@ -271,7 +297,7 @@ fn run_cancellable<'py>( pub(super) fn execute_worker<'py>( py: Python<'py>, - (snapshot, discovered): (&NativeOrchestrationSnapshot, Bound<'py, PyAny>), + (snapshot, discovered, blocked): (&NativeOrchestrationSnapshot, Bound<'py, PyAny>, NodeKeys), arguments: &KeywordArguments<'py>, (bearer, identity, recall): (&str, (String, String), Option<&InvocationRecall>), ) -> PyResult { @@ -286,7 +312,10 @@ pub(super) fn execute_worker<'py>( ) .map_err(|error| PyValueError::new_err(error.to_string()))?, )?; - let worker = WorkerBuild::new(&owner, (attached_nodes(snapshot), upstream_nodes(snapshot)))?; + let worker = WorkerBuild::new( + &owner, + (attached_nodes(snapshot), blocked, upstream_nodes(snapshot)), + )?; if let Some(recall) = recall { *recall .worker diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index b230cebb88..fc4f1f5988 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -123,6 +123,10 @@ class WorkerExecutionTestCase: concurrency_limits: dict[str, int] = field(default_factory=dict) launcher: Literal["native", "uncooperative", "docker"] = "native" warehouse_setup: Callable[[Path], None] = lambda _: None + # Each run node as (key, status, error code, skipped because of, error message). + expected_run_nodes: list[tuple[str, str, str | None, str | None, str | None]] = field( + default_factory=list + ) @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 7494f5ed13..2c286f5d48 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -232,7 +232,107 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail 'starts_at="2000-01-01T00:00:00Z"))\n' ), }, - ) + expected_run_nodes=[ + ("model:order_totals", "skipped", None, "model:stg_orders", None), + ("model:orders", "success", None, None, None), + ("model:raw_orders", "success", None, None, None), + ( + "model:stg_orders", + "failed", + "T003", + None, + "test 'stg_orders__doubles_amounts' failed for models: stg_orders", + ), + ], + ), + WorkerExecutionTestCase( + "a failing case of a parameterised test fails its model", + "SELECT 3 AS amount", + "SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('stg_orders', 'order_totals')", + "partial", + [(0,)], + files={ + "models/raw_orders.sql": ( + "MODEL (description 'Raw orders.', materialized table);\n" + "SELECT 1 AS order_id, 10 AS amount\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\n" + 'SELECT order_id, amount * 2 AS amount FROM __ref("raw_orders")\n' + ), + "models/order_totals.sql": ( + "MODEL (description 'Order totals.', materialized table);\n" + 'SELECT SUM(amount) AS total FROM __ref("stg_orders")\n' + ), + "tests/unit/test_stg_orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql" + ).read_text(), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + 'annual_orders = job("annual_orders", ' + 'select="orders raw_orders stg_orders order_totals", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + }, + expected_run_nodes=[ + ("model:order_totals", "skipped", None, "model:stg_orders", None), + ("model:orders", "success", None, None, None), + ("model:raw_orders", "success", None, None, None), + ( + "model:stg_orders", + "failed", + "T003", + None, + "test 'stg_orders__doubles_case_amounts [ten]' failed for models: stg_orders", + ), + ], + ), + WorkerExecutionTestCase( + "a failing assertion fails the model it reads, with no expected output", + "SELECT 3 AS amount", + "SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('stg_orders', 'order_totals')", + "partial", + [(0,)], + files={ + "models/raw_orders.sql": ( + "MODEL (description 'Raw orders.', materialized table);\n" + "SELECT 1 AS order_id, 10 AS amount\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\n" + 'SELECT order_id, amount * 2 AS amount FROM __ref("raw_orders")\n' + ), + "models/order_totals.sql": ( + "MODEL (description 'Order totals.', materialized table);\n" + 'SELECT SUM(amount) AS total FROM __ref("stg_orders")\n' + ), + "tests/unit/test_stg_orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_assert.sql" + ).read_text(), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + 'annual_orders = job("annual_orders", ' + 'select="orders raw_orders stg_orders order_totals", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + }, + expected_run_nodes=[ + ("model:order_totals", "skipped", None, "model:stg_orders", None), + ("model:orders", "success", None, None, None), + ("model:raw_orders", "success", None, None, None), + ( + "model:stg_orders", + "failed", + "T003", + None, + "test 'stg_orders__amounts_stay_small' failed for models: assertion amounts_stay_small", + ), + ], + ), ], ids=lambda case: case.description, ) @@ -253,18 +353,7 @@ def test_given_a_failing_unit_test_when_a_worker_builds_then_its_model_fails_and "WHERE n.run_id=? AND n.kind='node' ORDER BY n.node_key", (run["run_id"],), ).fetchall() - assert nodes == [ - ("model:order_totals", "skipped", None, "model:stg_orders", None), - ("model:orders", "success", None, None, None), - ("model:raw_orders", "success", None, None, None), - ( - "model:stg_orders", - "failed", - "T003", - None, - "test 'stg_orders__doubles_amounts' failed for models: stg_orders", - ), - ], worker.log.read_text() + assert nodes == test_case.expected_run_nodes, worker.log.read_text() with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows From e9e8ef2d21b642e0c734ad87b517b0d03a64b376 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:10:34 +0000 Subject: [PATCH 3/3] refactor(worker): read a test's blocked models from its compiled targets --- .../compiled_project/snapshot_execution.rs | 7 +++-- .../_helpers/location_command/constants.rs | 6 ----- .../_helpers/location_command/worker.rs | 26 +++++++------------ 3 files changed, 13 insertions(+), 26 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index 03abfdacc5..eadd9682fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -190,6 +190,7 @@ impl WorkerBuild { Ok(()) } + /// A skip in a build names the failed node upstream of it; a test or audit run that raised skips what it never checked, with no cause. fn complete( &self, py: Python<'_>, @@ -211,7 +212,6 @@ impl WorkerBuild { { return Ok(()); } - // A test or audit run that raised skips what it never checked, not what a failed check reads. let skipped_because_of = match outcome { NodeOutcome::Skipped => match self.execution.skipped_nodes.get(key) { Some(cause) => Some(cause.clone()), @@ -384,16 +384,15 @@ impl WorkerBuild { Ok(None) } - /// A failing SQL test blocks the models it covers, so they finish failed with its error rather than as bare skips. + /// A failing SQL test, found by its compiled name such as `parent [case]`, fails the models it blocks with its error. fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { let py = result.py(); - let outcome: String = field(result, "outcome")?.getattr("value")?.extract()?; + let outcome: String = field(result, "outcome")?.str()?.extract()?; if SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass) || self.worker.bind(py).get().refresh_cancellation(py)? { return Ok(()); } - // The snapshot keys a parameterised case by its compiled name, not by the file and block the result reports. let name: String = field(result, "test_name")?.extract()?; let key = format!("{SQL_TEST_KEY_PREFIX}{name}"); let error: Option = match result.getattr_opt("error_code")? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index 3095bcb0ea..b9e42eaa95 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -33,9 +33,3 @@ pub(crate) const REACH_SETTING: &str = "reach"; pub(crate) const JOB_RUN_REACH: &str = "all"; /// What a job run with `changed: True` counts when `[changes] counts` is not set: every change. pub(crate) const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; -/// The model lists of a compiled SQL test that a failure of it blocks in a build. -pub(crate) const BLOCKED_TEST_TARGETS: [&str; 3] = [ - "expected_model_names", - "assertion_target_model_names", - "reference_target_model_names", -]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index e1b76ad979..00681e4458 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -34,8 +34,7 @@ use crate::bindings::_helpers::compiled_views::project_reads::sql_tests; use crate::bindings::_helpers::discovery::project_config::configured_changes; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::constants::{ - BLOCKED_TEST_TARGETS, CHANGED_SETTING, COMMAND_IMPLEMENTATIONS, JOB_RUN_COUNTS, JOB_RUN_REACH, - REACH_SETTING, + CHANGED_SETTING, COMMAND_IMPLEMENTATIONS, JOB_RUN_COUNTS, JOB_RUN_REACH, REACH_SETTING, }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::recall::console::write_stderr; @@ -76,25 +75,20 @@ fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap) -> PyResult { let mut blocked: NodeKeys = BTreeMap::new(); for test in sql_tests(project)?.try_iter()? { let test = test?; let name: String = field(&field(&test, "key")?, "name")?.extract()?; - let mut models: Vec = Vec::new(); - for list in BLOCKED_TEST_TARGETS { - let Some(names) = test.getattr_opt(list)? else { - continue; - }; - for name in names.extract::>()? { - let model = format!("model:{name}"); - if !models.contains(&model) { - models.push(model); - } - } - } - blocked.insert(format!("{SQL_TEST_KEY_PREFIX}{name}"), models); + let models: Vec = field(&test, "target_model_names")?.extract()?; + blocked.insert( + format!("{SQL_TEST_KEY_PREFIX}{name}"), + models + .into_iter() + .map(|model| format!("model:{model}")) + .collect(), + ); } Ok(blocked) }