Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
TEST (name "udf__order_band__labels_amounts");

WITH input_values AS (
SELECT
v.amount
FROM VALUES
(5),
(50) AS v(amount)
),

__udf_actual__ AS (
SELECT
i.amount,
__udf("udf__order_band")(i.amount) AS band
FROM input_values AS i
),

__udf_expected__ AS (
SELECT
v.amount::INT AS amount,
v.band::VARCHAR AS band
FROM VALUES
(5, 'large'),
(50, 'large') AS v(amount, band)
)
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
TEST (name "stg_orders__doubles_case_amounts", parameters (amount integer), cases (ten (amount 10), twenty (amount 20)));
TEST (name "stg_orders__doubles_case_amounts", parameters (amount integer, doubled integer), cases (ten (amount 10, doubled 999), twenty (amount 20, doubled 40)));

WITH __ref__raw_orders AS (
SELECT 1 AS order_id, @param("amount") AS amount
),

__expected__stg_orders AS (
SELECT 1 AS order_id, 999 AS amount
SELECT 1 AS order_id, @param("doubled") AS amount
)

SELECT 1
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use std::sync::{Arc, Mutex};
use pyo3::exceptions::{PyKeyboardInterrupt, PyRuntimeError, PyValueError};
use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python};
use pyo3::types::PyStringMethods;
use sqlbuild_core::models::ResourceKind;
use sqlbuild_core::models::{CompiledResourceType, ResourceKind};
use sqlbuild_core::redaction::main::redact_error_text::redact_error_text;
use sqlbuild_host::models::SqlTestOutcome;
use sqlbuild_host::types::ResultEnum;
Expand All @@ -24,6 +24,9 @@ use crate::bindings::_helpers::records::fields::field;
/// Node keys by snapshot node key, such as the models each test checks.
pub(crate) type NodeKeys = BTreeMap<String, Vec<String>>;

/// A finished node's outcome, rows affected, error code and error message.
type Completion = (NodeOutcome, Option<i64>, (Option<String>, Option<String>));

/// This transient owner records only acknowledged outcomes; durable lifecycle remains on the API.
#[derive(Debug)]
pub(crate) struct WorkerBuild {
Expand All @@ -33,8 +36,12 @@ pub(crate) struct WorkerBuild {
checks: Mutex<BTreeMap<String, Option<NodeOutcome>>>,
/// The snapshot's audits and SQL tests, each with the node keys it checks.
attached: BTreeMap<String, Vec<String>>,
/// Each SQL test with every model a failure of it blocks in a build: its expected, assertion and reference targets.
/// Each SQL test with every node a failure of it blocks in a build: its expected, assertion and reference models and the functions it checks.
blocked: BTreeMap<String, Vec<String>>,
/// How many of each function's planned tests have still to report; a function is built before its tests run.
untested: Mutex<BTreeMap<String, usize>>,
/// A built function's result, held until its tests report so a failing one can fail it instead.
held: Mutex<BTreeMap<String, Completion>>,
/// Each snapshot node's direct upstream node keys, for naming what a skipped node waited on.
upstream: BTreeMap<String, Vec<String>>,
/// The failed node each finished node traces back to: itself when it failed, its cause when skipped.
Expand All @@ -59,6 +66,8 @@ impl WorkerBuild {
checks: Mutex::default(),
attached,
blocked,
untested: Mutex::default(),
held: Mutex::default(),
upstream,
causes: Mutex::default(),
checked: Mutex::default(),
Expand Down Expand Up @@ -137,6 +146,7 @@ impl WorkerBuild {
) -> Result<(), pyo3::PyErr> {
self.check_cancelled(pipeline.py())?;
let plan = field(pipeline, "plan_output")?;
self.count_planned_tests(&plan)?;
for collection in [
"model_entries",
"seed_entries",
Expand Down Expand Up @@ -171,6 +181,20 @@ impl WorkerBuild {
Ok(())
}

/// Count each function's tests this build plans, so a built function waits only for tests that will report.
fn count_planned_tests(&self, plan: &Bound<'_, PyAny>) -> PyResult<()> {
let mut untested = self.untested.lock().map_err(|_| poisoned())?;
for entry in field(plan, "test_entries")?.try_iter()? {
let test = format!("{SQL_TEST_KEY_PREFIX}{}", field(&entry?, "name")?.str()?);
for node in self.blocked.get(&test).into_iter().flatten() {
if is_function(node) {
*untested.entry(node.clone()).or_default() += 1;
}
}
}
Ok(())
}

/// Nodes `build --changed` leaves unbuilt are up to date, as a planner `no_change` skip is.
pub(crate) fn unchanged(&self, py: Python<'_>, keys: &[String]) -> PyResult<()> {
for key in keys {
Expand Down Expand Up @@ -389,35 +413,78 @@ impl WorkerBuild {
Ok(None)
}

/// A failing SQL test, found by its compiled name such as `parent [case]`, fails the models it blocks with its error.
/// A failing SQL test, found by its compiled name such as `parent [case]`, fails the nodes it blocks with its error; each test that reports releases a function waiting on its tests.
fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> {
let py = result.py();
let outcome: String = field(result, "outcome")?.str()?.extract()?;
if SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass)
|| self.worker.bind(py).get().refresh_cancellation(py)?
{
return Ok(());
}
let name: String = field(result, "test_name")?.extract()?;
let key = format!("{SQL_TEST_KEY_PREFIX}{name}");
let error: Option<String> = match result.getattr_opt("error_code")? {
Some(value) => value.extract()?,
None => None,
};
let message: Option<String> = match result.getattr_opt("error_message")? {
Some(value) => value.extract()?,
None => None,
};
for model in self.blocked.get(&key).into_iter().flatten() {
self.complete(
py,
model,
(NodeOutcome::Failed, None, (error.clone(), message.clone())),
)?;
if SqlTestOutcome::from_value(&outcome) != Some(SqlTestOutcome::Pass)
&& !self.worker.bind(py).get().refresh_cancellation(py)?
{
let error: Option<String> = match result.getattr_opt("error_code")? {
Some(value) => value.extract()?,
None => None,
};
let message: Option<String> = match result.getattr_opt("error_message")? {
Some(value) => value.extract()?,
None => None,
};
for node in self.blocked.get(&key).into_iter().flatten() {
self.complete(
py,
node,
(NodeOutcome::Failed, None, (error.clone(), message.clone())),
)?;
}
}
self.release_tested(py, &key)
}

/// Count `test` as reported for each function it checks, and publish a held function once none of its tests is left.
fn release_tested(&self, py: Python<'_>, test: &str) -> PyResult<()> {
let mut released: Vec<(String, Completion)> = Vec::new();
{
let mut untested = self.untested.lock().map_err(|_| poisoned())?;
let mut held = self.held.lock().map_err(|_| poisoned())?;
for node in self.blocked.get(test).into_iter().flatten() {
let Some(remaining) = untested.get_mut(node) else {
continue;
};
*remaining = remaining.saturating_sub(1);
if *remaining == 0
&& let Some(completion) = held.remove(node)
{
released.push((node.clone(), completion));
}
}
}
for (node, completion) in released {
self.complete(py, &node, completion)?;
}
Ok(())
}

/// Publish a node's result, or hold a built function's while its tests have still to report.
fn complete_or_hold(&self, py: Python<'_>, key: &str, completion: Completion) -> PyResult<()> {
let waiting = !self.checks_only()
&& matches!(completion.0, NodeOutcome::Success | NodeOutcome::Warn)
&& self
.untested
.lock()
.map_err(|_| poisoned())?
.get(key)
.is_some_and(|remaining| *remaining > 0);
if waiting {
self.held
.lock()
.map_err(|_| poisoned())?
.insert(key.into(), completion);
return Ok(());
}
self.complete(py, key, completion)
}

/// Existing native progress callbacks publish selected-node evidence without Python relays.
pub(crate) fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> {
if key.starts_with(SQL_TEST_KEY_PREFIX) {
Expand Down Expand Up @@ -466,7 +533,7 @@ impl WorkerBuild {
Some(value) => value.extract()?,
None => None,
};
self.complete(result.py(), key, (outcome, rows, (error, message)))?;
self.complete_or_hold(result.py(), key, (outcome, rows, (error, message)))?;
}
Ok(())
}
Expand Down Expand Up @@ -553,6 +620,13 @@ impl WorkerBuild {
}
}
}
let held: Vec<(String, Completion)> =
std::mem::take(&mut *self.held.lock().map_err(|_| poisoned())?)
.into_iter()
.collect();
for (key, completion) in held {
self.complete(py, &key, completion)?;
}
let returned = self.returned.load(Ordering::SeqCst);
let build = self.command().is_build();
let source_needs_no_run = |key: &str| build && key.starts_with("source:");
Expand Down Expand Up @@ -649,6 +723,15 @@ fn severity(outcome: NodeOutcome) -> u8 {
}
}

/// Whether a snapshot node key names a scalar or table function.
fn is_function(key: &str) -> bool {
key.split_once(':').is_some_and(|(kind, _)| {
[CompiledResourceType::Udf, CompiledResourceType::TableFn]
.iter()
.any(|function| function.as_str() == kind)
})
}

fn poisoned() -> pyo3::PyErr {
PyRuntimeError::new_err("worker build outcome lock is unavailable")
}
Original file line number Diff line number Diff line change
Expand Up @@ -75,20 +75,24 @@ fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap<String, Ve
attached
}

/// Each compiled SQL test with the models a failure of it blocks in a build: its expected, assertion and reference targets, as the scheduler's `scope_deps` names them.
/// Each compiled SQL test with the nodes a failure of it blocks in a build: its expected, assertion and reference models, as the scheduler's `scope_deps` names them, and the functions a direct test checks, keyed `udf:` or `table_fn:` like the snapshot.
pub(super) fn blocked_nodes(project: &Bound<'_, PyAny>) -> PyResult<NodeKeys> {
let mut blocked: NodeKeys = BTreeMap::new();
for test in sql_tests(project)?.try_iter()? {
let test = test?;
let name: String = field(&field(&test, "key")?, "name")?.extract()?;
let models: Vec<String> = field(&test, "target_model_names")?.extract()?;
blocked.insert(
format!("{SQL_TEST_KEY_PREFIX}{name}"),
models
.into_iter()
.map(|model| format!("model:{model}"))
.collect(),
);
let mut keys: Vec<String> = models
.into_iter()
.map(|model| format!("model:{model}"))
.collect();
for resource in field(&test, "tested_resources")?.try_iter()? {
let resource = resource?;
let kind: String = field(&resource, "kind")?.str()?.extract()?;
let name: String = field(&resource, "name")?.extract()?;
keys.push(format!("{kind}:{name}"));
}
blocked.insert(format!("{SQL_TEST_KEY_PREFIX}{name}"), keys);
}
Ok(blocked)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,103 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail
),
],
),
WorkerExecutionTestCase(
"a failing function test fails its function, and the models calling it are skipped because of it",
"SELECT 3 AS amount",
"SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('stg_orders', 'order_totals')",
"partial",
[(0,)],
files={
"functions/sql/udf__order_band.sql": (
"FUNCTION (description 'Size band of an order amount.', "
"arguments (amount INT), returns VARCHAR);\n"
"CASE WHEN amount >= 20 THEN 'large' ELSE 'small' END\n"
),
"models/raw_orders.sql": (
"MODEL (description 'Raw orders.', materialized table);\n"
"SELECT 1 AS order_id, 10 AS amount\n"
),
"models/stg_orders.sql": (
"MODEL (description 'Staged orders.', materialized table);\n"
'SELECT order_id, __udf("udf__order_band")(amount) AS band FROM __ref("raw_orders")\n'
),
"models/order_totals.sql": (
"MODEL (description 'Order totals.', materialized table);\n"
'SELECT band, COUNT(*) AS orders FROM __ref("stg_orders") GROUP BY band\n'
),
"tests/unit/test_udf__order_band.sql": Path(
"benchmarks/duckshop/edge_cases/orchestration/worker_failed_function_test.sql"
).read_text(),
"orchestration/orders.py": (
"from sqlbuild.orchestration import job, schedule\n"
'annual_orders = job("annual_orders", '
'select="orders raw_orders udf__order_band stg_orders order_totals", '
'description="Build orders.", '
'schedule=schedule(cron="0 0 * * *", timezone="UTC", '
'starts_at="2000-01-01T00:00:00Z"))\n'
),
},
expected_run_nodes=[
("model:order_totals", "skipped", None, "udf:udf__order_band", None),
("model:orders", "success", None, None, None),
("model:raw_orders", "success", None, None, None),
("model:stg_orders", "skipped", None, "udf:udf__order_band", None),
(
"udf:udf__order_band",
"failed",
"T003",
None,
"test 'udf__order_band__labels_amounts' failed for models: udf "
"udf__order_band__labels_amounts",
),
],
),
WorkerExecutionTestCase(
"a passing function test releases its function, and the models calling it build",
"SELECT 3 AS amount",
"SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('stg_orders', 'order_totals')",
"success",
[(2,)],
files={
"functions/sql/udf__order_band.sql": (
"FUNCTION (description 'Size band of an order amount.', "
"arguments (amount INT), returns VARCHAR);\n"
"CASE WHEN amount >= 20 THEN 'large' ELSE 'small' END\n"
),
"models/raw_orders.sql": (
"MODEL (description 'Raw orders.', materialized table);\n"
"SELECT 1 AS order_id, 10 AS amount\n"
),
"models/stg_orders.sql": (
"MODEL (description 'Staged orders.', materialized table);\n"
'SELECT order_id, __udf("udf__order_band")(amount) AS band FROM __ref("raw_orders")\n'
),
"models/order_totals.sql": (
"MODEL (description 'Order totals.', materialized table);\n"
'SELECT band, COUNT(*) AS orders FROM __ref("stg_orders") GROUP BY band\n'
),
"tests/unit/test_udf__order_band.sql": Path(
"benchmarks/duckshop/edge_cases/orchestration/worker_failed_function_test.sql"
)
.read_text()
.replace("(5, 'large')", "(5, 'small')"),
"orchestration/orders.py": (
"from sqlbuild.orchestration import job, schedule\n"
'annual_orders = job("annual_orders", '
'select="orders raw_orders udf__order_band stg_orders order_totals", '
'description="Build orders.", '
'schedule=schedule(cron="0 0 * * *", timezone="UTC", '
'starts_at="2000-01-01T00:00:00Z"))\n'
),
},
expected_run_nodes=[
("model:order_totals", "success", None, None, None),
("model:orders", "success", None, None, None),
("model:raw_orders", "success", None, None, None),
("model:stg_orders", "success", None, None, None),
("udf:udf__order_band", "success", None, None, None),
],
),
],
ids=lambda case: case.description,
)
Expand Down
Loading