From 8f7d0a927cb5ab5383e9483bd847f896ae54a093 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:01:58 +0000 Subject: [PATCH 1/2] fix(worker): fail a function whose unit test fails, and name it on downstream skips A function is built before its unit tests run, so a worker build reported it as a success even when a test then failed, and the models calling it were skipped with no cause. The worker now holds a built function's result until every test that checks it has reported. A failing test records the function as failed with the test's error, so the run page, the Failures list and "Rerun from failure" point at it. Results still held when the build ends are published as they were. --- .../worker_failed_function_test.sql | 25 ++++ .../compiled_project/snapshot_execution.rs | 129 ++++++++++++++---- .../_helpers/location_command/worker.rs | 20 +-- .../publication/test_worker.py | 97 +++++++++++++ 4 files changed, 240 insertions(+), 31 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/worker_failed_function_test.sql diff --git a/benchmarks/duckshop/edge_cases/orchestration/worker_failed_function_test.sql b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_function_test.sql new file mode 100644 index 0000000000..775ab2eba7 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_function_test.sql @@ -0,0 +1,25 @@ +TEST (name "udf__order_band__labels_amounts"); + +WITH input_values AS ( + SELECT + v.amount + FROM VALUES + (5), + (50) AS v(amount) +), + +__udf_actual__ AS ( + SELECT + i.amount, + __udf("udf__order_band")(i.amount) AS band + FROM input_values AS i +), + +__udf_expected__ AS ( + SELECT + v.amount::INT AS amount, + v.band::VARCHAR AS band + FROM VALUES + (5, 'large'), + (50, 'large') AS v(amount, band) +) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index cf6fa52ee7..f20b858789 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, Mutex}; use pyo3::exceptions::{PyKeyboardInterrupt, PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyStringMethods; -use sqlbuild_core::models::ResourceKind; +use sqlbuild_core::models::{CompiledResourceType, ResourceKind}; use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; use sqlbuild_host::models::SqlTestOutcome; use sqlbuild_host::types::ResultEnum; @@ -24,6 +24,9 @@ use crate::bindings::_helpers::records::fields::field; /// Node keys by snapshot node key, such as the models each test checks. pub(crate) type NodeKeys = BTreeMap>; +/// A finished node's outcome, rows affected, error code and error message. +type Completion = (NodeOutcome, Option, (Option, Option)); + /// This transient owner records only acknowledged outcomes; durable lifecycle remains on the API. #[derive(Debug)] pub(crate) struct WorkerBuild { @@ -33,8 +36,12 @@ pub(crate) struct WorkerBuild { checks: Mutex>>, /// The snapshot's audits and SQL tests, each with the node keys it checks. attached: BTreeMap>, - /// Each SQL test with every model a failure of it blocks in a build: its expected, assertion and reference targets. + /// Each SQL test with every node a failure of it blocks in a build: its expected, assertion and reference models and the functions it checks. blocked: BTreeMap>, + /// How many of each function's planned tests have still to report; a function is built before its tests run. + untested: Mutex>, + /// A built function's result, held until its tests report so a failing one can fail it instead. + held: Mutex>, /// Each snapshot node's direct upstream node keys, for naming what a skipped node waited on. upstream: BTreeMap>, /// The failed node each finished node traces back to: itself when it failed, its cause when skipped. @@ -59,6 +66,8 @@ impl WorkerBuild { checks: Mutex::default(), attached, blocked, + untested: Mutex::default(), + held: Mutex::default(), upstream, causes: Mutex::default(), checked: Mutex::default(), @@ -137,6 +146,7 @@ impl WorkerBuild { ) -> Result<(), pyo3::PyErr> { self.check_cancelled(pipeline.py())?; let plan = field(pipeline, "plan_output")?; + self.count_planned_tests(&plan)?; for collection in [ "model_entries", "seed_entries", @@ -171,6 +181,20 @@ impl WorkerBuild { Ok(()) } + /// Count each function's tests this build plans, so a built function waits only for tests that will report. + fn count_planned_tests(&self, plan: &Bound<'_, PyAny>) -> PyResult<()> { + let mut untested = self.untested.lock().map_err(|_| poisoned())?; + for entry in field(plan, "test_entries")?.try_iter()? { + let test = format!("{SQL_TEST_KEY_PREFIX}{}", field(&entry?, "name")?.str()?); + for node in self.blocked.get(&test).into_iter().flatten() { + if is_function(node) { + *untested.entry(node.clone()).or_default() += 1; + } + } + } + Ok(()) + } + /// Nodes `build --changed` leaves unbuilt are up to date, as a planner `no_change` skip is. pub(crate) fn unchanged(&self, py: Python<'_>, keys: &[String]) -> PyResult<()> { for key in keys { @@ -389,35 +413,78 @@ impl WorkerBuild { Ok(None) } - /// A failing SQL test, found by its compiled name such as `parent [case]`, fails the models it blocks with its error. + /// A failing SQL test, found by its compiled name such as `parent [case]`, fails the nodes it blocks with its error; each test that reports releases a function waiting on its tests. fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { let py = result.py(); let outcome: String = field(result, "outcome")?.str()?.extract()?; - if SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass) - || self.worker.bind(py).get().refresh_cancellation(py)? - { - return Ok(()); - } let name: String = field(result, "test_name")?.extract()?; let key = format!("{SQL_TEST_KEY_PREFIX}{name}"); - let error: Option = match result.getattr_opt("error_code")? { - Some(value) => value.extract()?, - None => None, - }; - let message: Option = match result.getattr_opt("error_message")? { - Some(value) => value.extract()?, - None => None, - }; - for model in self.blocked.get(&key).into_iter().flatten() { - self.complete( - py, - model, - (NodeOutcome::Failed, None, (error.clone(), message.clone())), - )?; + if SqlTestOutcome::from_value(&outcome) != Some(SqlTestOutcome::Pass) + && !self.worker.bind(py).get().refresh_cancellation(py)? + { + let error: Option = match result.getattr_opt("error_code")? { + Some(value) => value.extract()?, + None => None, + }; + let message: Option = match result.getattr_opt("error_message")? { + Some(value) => value.extract()?, + None => None, + }; + for node in self.blocked.get(&key).into_iter().flatten() { + self.complete( + py, + node, + (NodeOutcome::Failed, None, (error.clone(), message.clone())), + )?; + } + } + self.release_tested(py, &key) + } + + /// Count `test` as reported for each function it checks, and publish a held function once none of its tests is left. + fn release_tested(&self, py: Python<'_>, test: &str) -> PyResult<()> { + let mut released: Vec<(String, Completion)> = Vec::new(); + { + let mut untested = self.untested.lock().map_err(|_| poisoned())?; + let mut held = self.held.lock().map_err(|_| poisoned())?; + for node in self.blocked.get(test).into_iter().flatten() { + let Some(remaining) = untested.get_mut(node) else { + continue; + }; + *remaining = remaining.saturating_sub(1); + if *remaining == 0 + && let Some(completion) = held.remove(node) + { + released.push((node.clone(), completion)); + } + } + } + for (node, completion) in released { + self.complete(py, &node, completion)?; } Ok(()) } + /// Publish a node's result, or hold a built function's while its tests have still to report. + fn complete_or_hold(&self, py: Python<'_>, key: &str, completion: Completion) -> PyResult<()> { + let waiting = !self.checks_only() + && matches!(completion.0, NodeOutcome::Success | NodeOutcome::Warn) + && self + .untested + .lock() + .map_err(|_| poisoned())? + .get(key) + .is_some_and(|remaining| *remaining > 0); + if waiting { + self.held + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), completion); + return Ok(()); + } + self.complete(py, key, completion) + } + /// Existing native progress callbacks publish selected-node evidence without Python relays. pub(crate) fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { if key.starts_with(SQL_TEST_KEY_PREFIX) { @@ -466,7 +533,7 @@ impl WorkerBuild { Some(value) => value.extract()?, None => None, }; - self.complete(result.py(), key, (outcome, rows, (error, message)))?; + self.complete_or_hold(result.py(), key, (outcome, rows, (error, message)))?; } Ok(()) } @@ -553,6 +620,13 @@ impl WorkerBuild { } } } + let held: Vec<(String, Completion)> = + std::mem::take(&mut *self.held.lock().map_err(|_| poisoned())?) + .into_iter() + .collect(); + for (key, completion) in held { + self.complete(py, &key, completion)?; + } let returned = self.returned.load(Ordering::SeqCst); let build = self.command().is_build(); let source_needs_no_run = |key: &str| build && key.starts_with("source:"); @@ -649,6 +723,15 @@ fn severity(outcome: NodeOutcome) -> u8 { } } +/// Whether a snapshot node key names a scalar or table function. +fn is_function(key: &str) -> bool { + key.split_once(':').is_some_and(|(kind, _)| { + [CompiledResourceType::Udf, CompiledResourceType::TableFn] + .iter() + .any(|function| function.as_str() == kind) + }) +} + fn poisoned() -> pyo3::PyErr { PyRuntimeError::new_err("worker build outcome lock is unavailable") } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 00681e4458..df3da90b96 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -75,20 +75,24 @@ fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap) -> PyResult { let mut blocked: NodeKeys = BTreeMap::new(); for test in sql_tests(project)?.try_iter()? { let test = test?; let name: String = field(&field(&test, "key")?, "name")?.extract()?; let models: Vec = field(&test, "target_model_names")?.extract()?; - blocked.insert( - format!("{SQL_TEST_KEY_PREFIX}{name}"), - models - .into_iter() - .map(|model| format!("model:{model}")) - .collect(), - ); + let mut keys: Vec = models + .into_iter() + .map(|model| format!("model:{model}")) + .collect(); + for resource in field(&test, "tested_resources")?.try_iter()? { + let resource = resource?; + let kind: String = field(&resource, "kind")?.str()?.extract()?; + let name: String = field(&resource, "name")?.extract()?; + keys.push(format!("{kind}:{name}")); + } + blocked.insert(format!("{SQL_TEST_KEY_PREFIX}{name}"), keys); } Ok(blocked) } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 0efad2b17f..fd3cb32046 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -333,6 +333,103 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail ), ], ), + WorkerExecutionTestCase( + "a failing function test fails its function, and the models calling it are skipped because of it", + "SELECT 3 AS amount", + "SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('stg_orders', 'order_totals')", + "partial", + [(0,)], + files={ + "functions/sql/udf__order_band.sql": ( + "FUNCTION (description 'Size band of an order amount.', " + "arguments (amount INT), returns VARCHAR);\n" + "CASE WHEN amount >= 20 THEN 'large' ELSE 'small' END\n" + ), + "models/raw_orders.sql": ( + "MODEL (description 'Raw orders.', materialized table);\n" + "SELECT 1 AS order_id, 10 AS amount\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\n" + 'SELECT order_id, __udf("udf__order_band")(amount) AS band FROM __ref("raw_orders")\n' + ), + "models/order_totals.sql": ( + "MODEL (description 'Order totals.', materialized table);\n" + 'SELECT band, COUNT(*) AS orders FROM __ref("stg_orders") GROUP BY band\n' + ), + "tests/unit/test_udf__order_band.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/worker_failed_function_test.sql" + ).read_text(), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + 'annual_orders = job("annual_orders", ' + 'select="orders raw_orders udf__order_band stg_orders order_totals", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + }, + expected_run_nodes=[ + ("model:order_totals", "skipped", None, "udf:udf__order_band", None), + ("model:orders", "success", None, None, None), + ("model:raw_orders", "success", None, None, None), + ("model:stg_orders", "skipped", None, "udf:udf__order_band", None), + ( + "udf:udf__order_band", + "failed", + "T003", + None, + "test 'udf__order_band__labels_amounts' failed for models: udf " + "udf__order_band__labels_amounts", + ), + ], + ), + WorkerExecutionTestCase( + "a passing function test releases its function, and the models calling it build", + "SELECT 3 AS amount", + "SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('stg_orders', 'order_totals')", + "success", + [(2,)], + files={ + "functions/sql/udf__order_band.sql": ( + "FUNCTION (description 'Size band of an order amount.', " + "arguments (amount INT), returns VARCHAR);\n" + "CASE WHEN amount >= 20 THEN 'large' ELSE 'small' END\n" + ), + "models/raw_orders.sql": ( + "MODEL (description 'Raw orders.', materialized table);\n" + "SELECT 1 AS order_id, 10 AS amount\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\n" + 'SELECT order_id, __udf("udf__order_band")(amount) AS band FROM __ref("raw_orders")\n' + ), + "models/order_totals.sql": ( + "MODEL (description 'Order totals.', materialized table);\n" + 'SELECT band, COUNT(*) AS orders FROM __ref("stg_orders") GROUP BY band\n' + ), + "tests/unit/test_udf__order_band.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/worker_failed_function_test.sql" + ) + .read_text() + .replace("(5, 'large')", "(5, 'small')"), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + 'annual_orders = job("annual_orders", ' + 'select="orders raw_orders udf__order_band stg_orders order_totals", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + }, + expected_run_nodes=[ + ("model:order_totals", "success", None, None, None), + ("model:orders", "success", None, None, None), + ("model:raw_orders", "success", None, None, None), + ("model:stg_orders", "success", None, None, None), + ("udf:udf__order_band", "success", None, None, None), + ], + ), ], ids=lambda case: case.description, ) From 204279eb00069fec48f525cb028184f2cdaff454 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:54:58 +0000 Subject: [PATCH 2/2] test(worker): let only one case of the parameterised unit test fail Both cases failed, so which one the worker recorded against the model depended on execution order. --- .../orchestration/worker_failed_unit_test_cases.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql index 41cc2593cc..d8c6d559d1 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql +++ b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql @@ -1,11 +1,11 @@ -TEST (name "stg_orders__doubles_case_amounts", parameters (amount integer), cases (ten (amount 10), twenty (amount 20))); +TEST (name "stg_orders__doubles_case_amounts", parameters (amount integer, doubled integer), cases (ten (amount 10, doubled 999), twenty (amount 20, doubled 40))); WITH __ref__raw_orders AS ( SELECT 1 AS order_id, @param("amount") AS amount ), __expected__stg_orders AS ( - SELECT 1 AS order_id, 999 AS amount + SELECT 1 AS order_id, @param("doubled") AS amount ) SELECT 1