diff --git a/benchmarks/duckshop/edge_cases/orchestration/worker_failed_function_test.sql b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_function_test.sql new file mode 100644 index 0000000000..775ab2eba7 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_function_test.sql @@ -0,0 +1,25 @@ +TEST (name "udf__order_band__labels_amounts"); + +WITH input_values AS ( + SELECT + v.amount + FROM VALUES + (5), + (50) AS v(amount) +), + +__udf_actual__ AS ( + SELECT + i.amount, + __udf("udf__order_band")(i.amount) AS band + FROM input_values AS i +), + +__udf_expected__ AS ( + SELECT + v.amount::INT AS amount, + v.band::VARCHAR AS band + FROM VALUES + (5, 'large'), + (50, 'large') AS v(amount, band) +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql index 41cc2593cc..d8c6d559d1 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql +++ b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql @@ -1,11 +1,11 @@ -TEST (name "stg_orders__doubles_case_amounts", parameters (amount integer), cases (ten (amount 10), twenty (amount 20))); +TEST (name "stg_orders__doubles_case_amounts", parameters (amount integer, doubled integer), cases (ten (amount 10, doubled 999), twenty (amount 20, doubled 40))); WITH __ref__raw_orders AS ( SELECT 1 AS order_id, @param("amount") AS amount ), __expected__stg_orders AS ( - SELECT 1 AS order_id, 999 AS amount + SELECT 1 AS order_id, @param("doubled") AS amount ) SELECT 1 diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index cf6fa52ee7..f20b858789 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, Mutex}; use pyo3::exceptions::{PyKeyboardInterrupt, PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyStringMethods; -use sqlbuild_core::models::ResourceKind; +use sqlbuild_core::models::{CompiledResourceType, ResourceKind}; use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; use sqlbuild_host::models::SqlTestOutcome; use sqlbuild_host::types::ResultEnum; @@ -24,6 +24,9 @@ use crate::bindings::_helpers::records::fields::field; /// Node keys by snapshot node key, such as the models each test checks. pub(crate) type NodeKeys = BTreeMap>; +/// A finished node's outcome, rows affected, error code and error message. +type Completion = (NodeOutcome, Option, (Option, Option)); + /// This transient owner records only acknowledged outcomes; durable lifecycle remains on the API. #[derive(Debug)] pub(crate) struct WorkerBuild { @@ -33,8 +36,12 @@ pub(crate) struct WorkerBuild { checks: Mutex>>, /// The snapshot's audits and SQL tests, each with the node keys it checks. attached: BTreeMap>, - /// Each SQL test with every model a failure of it blocks in a build: its expected, assertion and reference targets. + /// Each SQL test with every node a failure of it blocks in a build: its expected, assertion and reference models and the functions it checks. blocked: BTreeMap>, + /// How many of each function's planned tests have still to report; a function is built before its tests run. + untested: Mutex>, + /// A built function's result, held until its tests report so a failing one can fail it instead. + held: Mutex>, /// Each snapshot node's direct upstream node keys, for naming what a skipped node waited on. upstream: BTreeMap>, /// The failed node each finished node traces back to: itself when it failed, its cause when skipped. @@ -59,6 +66,8 @@ impl WorkerBuild { checks: Mutex::default(), attached, blocked, + untested: Mutex::default(), + held: Mutex::default(), upstream, causes: Mutex::default(), checked: Mutex::default(), @@ -137,6 +146,7 @@ impl WorkerBuild { ) -> Result<(), pyo3::PyErr> { self.check_cancelled(pipeline.py())?; let plan = field(pipeline, "plan_output")?; + self.count_planned_tests(&plan)?; for collection in [ "model_entries", "seed_entries", @@ -171,6 +181,20 @@ impl WorkerBuild { Ok(()) } + /// Count each function's tests this build plans, so a built function waits only for tests that will report. + fn count_planned_tests(&self, plan: &Bound<'_, PyAny>) -> PyResult<()> { + let mut untested = self.untested.lock().map_err(|_| poisoned())?; + for entry in field(plan, "test_entries")?.try_iter()? { + let test = format!("{SQL_TEST_KEY_PREFIX}{}", field(&entry?, "name")?.str()?); + for node in self.blocked.get(&test).into_iter().flatten() { + if is_function(node) { + *untested.entry(node.clone()).or_default() += 1; + } + } + } + Ok(()) + } + /// Nodes `build --changed` leaves unbuilt are up to date, as a planner `no_change` skip is. pub(crate) fn unchanged(&self, py: Python<'_>, keys: &[String]) -> PyResult<()> { for key in keys { @@ -389,35 +413,78 @@ impl WorkerBuild { Ok(None) } - /// A failing SQL test, found by its compiled name such as `parent [case]`, fails the models it blocks with its error. + /// A failing SQL test, found by its compiled name such as `parent [case]`, fails the nodes it blocks with its error; each test that reports releases a function waiting on its tests. fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { let py = result.py(); let outcome: String = field(result, "outcome")?.str()?.extract()?; - if SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass) - || self.worker.bind(py).get().refresh_cancellation(py)? - { - return Ok(()); - } let name: String = field(result, "test_name")?.extract()?; let key = format!("{SQL_TEST_KEY_PREFIX}{name}"); - let error: Option = match result.getattr_opt("error_code")? { - Some(value) => value.extract()?, - None => None, - }; - let message: Option = match result.getattr_opt("error_message")? { - Some(value) => value.extract()?, - None => None, - }; - for model in self.blocked.get(&key).into_iter().flatten() { - self.complete( - py, - model, - (NodeOutcome::Failed, None, (error.clone(), message.clone())), - )?; + if SqlTestOutcome::from_value(&outcome) != Some(SqlTestOutcome::Pass) + && !self.worker.bind(py).get().refresh_cancellation(py)? + { + let error: Option = match result.getattr_opt("error_code")? { + Some(value) => value.extract()?, + None => None, + }; + let message: Option = match result.getattr_opt("error_message")? { + Some(value) => value.extract()?, + None => None, + }; + for node in self.blocked.get(&key).into_iter().flatten() { + self.complete( + py, + node, + (NodeOutcome::Failed, None, (error.clone(), message.clone())), + )?; + } + } + self.release_tested(py, &key) + } + + /// Count `test` as reported for each function it checks, and publish a held function once none of its tests is left. + fn release_tested(&self, py: Python<'_>, test: &str) -> PyResult<()> { + let mut released: Vec<(String, Completion)> = Vec::new(); + { + let mut untested = self.untested.lock().map_err(|_| poisoned())?; + let mut held = self.held.lock().map_err(|_| poisoned())?; + for node in self.blocked.get(test).into_iter().flatten() { + let Some(remaining) = untested.get_mut(node) else { + continue; + }; + *remaining = remaining.saturating_sub(1); + if *remaining == 0 + && let Some(completion) = held.remove(node) + { + released.push((node.clone(), completion)); + } + } + } + for (node, completion) in released { + self.complete(py, &node, completion)?; } Ok(()) } + /// Publish a node's result, or hold a built function's while its tests have still to report. + fn complete_or_hold(&self, py: Python<'_>, key: &str, completion: Completion) -> PyResult<()> { + let waiting = !self.checks_only() + && matches!(completion.0, NodeOutcome::Success | NodeOutcome::Warn) + && self + .untested + .lock() + .map_err(|_| poisoned())? + .get(key) + .is_some_and(|remaining| *remaining > 0); + if waiting { + self.held + .lock() + .map_err(|_| poisoned())? + .insert(key.into(), completion); + return Ok(()); + } + self.complete(py, key, completion) + } + /// Existing native progress callbacks publish selected-node evidence without Python relays. pub(crate) fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> { if key.starts_with(SQL_TEST_KEY_PREFIX) { @@ -466,7 +533,7 @@ impl WorkerBuild { Some(value) => value.extract()?, None => None, }; - self.complete(result.py(), key, (outcome, rows, (error, message)))?; + self.complete_or_hold(result.py(), key, (outcome, rows, (error, message)))?; } Ok(()) } @@ -553,6 +620,13 @@ impl WorkerBuild { } } } + let held: Vec<(String, Completion)> = + std::mem::take(&mut *self.held.lock().map_err(|_| poisoned())?) + .into_iter() + .collect(); + for (key, completion) in held { + self.complete(py, &key, completion)?; + } let returned = self.returned.load(Ordering::SeqCst); let build = self.command().is_build(); let source_needs_no_run = |key: &str| build && key.starts_with("source:"); @@ -649,6 +723,15 @@ fn severity(outcome: NodeOutcome) -> u8 { } } +/// Whether a snapshot node key names a scalar or table function. +fn is_function(key: &str) -> bool { + key.split_once(':').is_some_and(|(kind, _)| { + [CompiledResourceType::Udf, CompiledResourceType::TableFn] + .iter() + .any(|function| function.as_str() == kind) + }) +} + fn poisoned() -> pyo3::PyErr { PyRuntimeError::new_err("worker build outcome lock is unavailable") } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 00681e4458..df3da90b96 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -75,20 +75,24 @@ fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap) -> PyResult { let mut blocked: NodeKeys = BTreeMap::new(); for test in sql_tests(project)?.try_iter()? { let test = test?; let name: String = field(&field(&test, "key")?, "name")?.extract()?; let models: Vec = field(&test, "target_model_names")?.extract()?; - blocked.insert( - format!("{SQL_TEST_KEY_PREFIX}{name}"), - models - .into_iter() - .map(|model| format!("model:{model}")) - .collect(), - ); + let mut keys: Vec = models + .into_iter() + .map(|model| format!("model:{model}")) + .collect(); + for resource in field(&test, "tested_resources")?.try_iter()? { + let resource = resource?; + let kind: String = field(&resource, "kind")?.str()?.extract()?; + let name: String = field(&resource, "name")?.extract()?; + keys.push(format!("{kind}:{name}")); + } + blocked.insert(format!("{SQL_TEST_KEY_PREFIX}{name}"), keys); } Ok(blocked) } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 0efad2b17f..fd3cb32046 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -333,6 +333,103 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail ), ], ), + WorkerExecutionTestCase( + "a failing function test fails its function, and the models calling it are skipped because of it", + "SELECT 3 AS amount", + "SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('stg_orders', 'order_totals')", + "partial", + [(0,)], + files={ + "functions/sql/udf__order_band.sql": ( + "FUNCTION (description 'Size band of an order amount.', " + "arguments (amount INT), returns VARCHAR);\n" + "CASE WHEN amount >= 20 THEN 'large' ELSE 'small' END\n" + ), + "models/raw_orders.sql": ( + "MODEL (description 'Raw orders.', materialized table);\n" + "SELECT 1 AS order_id, 10 AS amount\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\n" + 'SELECT order_id, __udf("udf__order_band")(amount) AS band FROM __ref("raw_orders")\n' + ), + "models/order_totals.sql": ( + "MODEL (description 'Order totals.', materialized table);\n" + 'SELECT band, COUNT(*) AS orders FROM __ref("stg_orders") GROUP BY band\n' + ), + "tests/unit/test_udf__order_band.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/worker_failed_function_test.sql" + ).read_text(), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + 'annual_orders = job("annual_orders", ' + 'select="orders raw_orders udf__order_band stg_orders order_totals", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + }, + expected_run_nodes=[ + ("model:order_totals", "skipped", None, "udf:udf__order_band", None), + ("model:orders", "success", None, None, None), + ("model:raw_orders", "success", None, None, None), + ("model:stg_orders", "skipped", None, "udf:udf__order_band", None), + ( + "udf:udf__order_band", + "failed", + "T003", + None, + "test 'udf__order_band__labels_amounts' failed for models: udf " + "udf__order_band__labels_amounts", + ), + ], + ), + WorkerExecutionTestCase( + "a passing function test releases its function, and the models calling it build", + "SELECT 3 AS amount", + "SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('stg_orders', 'order_totals')", + "success", + [(2,)], + files={ + "functions/sql/udf__order_band.sql": ( + "FUNCTION (description 'Size band of an order amount.', " + "arguments (amount INT), returns VARCHAR);\n" + "CASE WHEN amount >= 20 THEN 'large' ELSE 'small' END\n" + ), + "models/raw_orders.sql": ( + "MODEL (description 'Raw orders.', materialized table);\n" + "SELECT 1 AS order_id, 10 AS amount\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\n" + 'SELECT order_id, __udf("udf__order_band")(amount) AS band FROM __ref("raw_orders")\n' + ), + "models/order_totals.sql": ( + "MODEL (description 'Order totals.', materialized table);\n" + 'SELECT band, COUNT(*) AS orders FROM __ref("stg_orders") GROUP BY band\n' + ), + "tests/unit/test_udf__order_band.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/worker_failed_function_test.sql" + ) + .read_text() + .replace("(5, 'large')", "(5, 'small')"), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + 'annual_orders = job("annual_orders", ' + 'select="orders raw_orders udf__order_band stg_orders order_totals", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + }, + expected_run_nodes=[ + ("model:order_totals", "success", None, None, None), + ("model:orders", "success", None, None, None), + ("model:raw_orders", "success", None, None, None), + ("model:stg_orders", "success", None, None, None), + ("udf:udf__order_band", "success", None, None, None), + ], + ), ], ids=lambda case: case.description, )