Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
TEST (name "stg_orders__doubles_amounts");

WITH __ref__raw_orders AS (
SELECT 1 AS order_id, 10 AS amount
),

__expected__stg_orders AS (
SELECT 1 AS order_id, 999 AS amount
)

SELECT 1
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
TEST (name "stg_orders__amounts_stay_small");

WITH __ref__raw_orders AS (
SELECT 1 AS order_id, 10 AS amount
),

__assert__amounts_stay_small AS (
SELECT order_id FROM __ref("stg_orders") WHERE amount > 5
)

SELECT 1
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
TEST (name "stg_orders__doubles_case_amounts", parameters (amount integer), cases (ten (amount 10), twenty (amount 20)));

WITH __ref__raw_orders AS (
SELECT 1 AS order_id, @param("amount") AS amount
),

__expected__stg_orders AS (
SELECT 1 AS order_id, 999 AS amount
)

SELECT 1
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,5 @@ pub(crate) const STORE_BUSY_STATUS: u16 = 503;
pub(crate) const JOB_SELECTOR_KINDS: [&str; 2] = ["job", "name"];
/// Separates the parts of an intersection selector.
pub(crate) const SELECTOR_INTERSECTION: char = ',';
/// The key prefix of a SQL unit test's result, as the snapshot names it.
pub(crate) const SQL_TEST_KEY_PREFIX: &str = "sql_test:";
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//! Frozen selection, executor callbacks and completion belong to one verified worker build.

use std::collections::BTreeMap;
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};

Expand All @@ -9,14 +9,21 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python};
use pyo3::types::PyStringMethods;
use sqlbuild_core::models::ResourceKind;
use sqlbuild_core::redaction::main::redact_error_text::redact_error_text;
use sqlbuild_host::models::SqlTestOutcome;
use sqlbuild_host::types::ResultEnum;
use sqlbuild_orchestration::runs::models::{
NodeOutcome, RunChange, RunCommand, RunExecution, RunOutcome,
};

use crate::bindings::_helpers::compiled_project::constants::{NO_CHANGE_REASON, SKIP_ACTION};
use crate::bindings::_helpers::compiled_project::constants::{
NO_CHANGE_REASON, SKIP_ACTION, SQL_TEST_KEY_PREFIX,
};
use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker;
use crate::bindings::_helpers::records::fields::field;

/// Node keys by snapshot node key, such as the models each test checks.
pub(crate) type NodeKeys = BTreeMap<String, Vec<String>>;

/// This transient owner records only acknowledged outcomes; durable lifecycle remains on the API.
#[derive(Debug)]
pub(crate) struct WorkerBuild {
Expand All @@ -26,6 +33,12 @@ pub(crate) struct WorkerBuild {
checks: Mutex<BTreeMap<String, Option<NodeOutcome>>>,
/// The snapshot's audits and SQL tests, each with the node keys it checks.
attached: BTreeMap<String, Vec<String>>,
/// Each SQL test with every model a failure of it blocks in a build: its expected, assertion and reference targets.
blocked: BTreeMap<String, Vec<String>>,
/// Each snapshot node's direct upstream node keys, for naming what a skipped node waited on.
upstream: BTreeMap<String, Vec<String>>,
/// The failed node each finished node traces back to: itself when it failed, its cause when skipped.
causes: Mutex<BTreeMap<String, String>>,
/// The worst attached outcome of each node, for commands that only run tests or audits.
checked: Mutex<BTreeMap<String, NodeOutcome>>,
/// Whether the command returned, having run every check it planned, rather than being refused or raising.
Expand All @@ -37,14 +50,17 @@ pub(crate) struct WorkerBuild {
impl WorkerBuild {
pub(crate) fn new(
worker: &Bound<'_, NativeOrchestrationWorker>,
attached: BTreeMap<String, Vec<String>>,
(attached, blocked, upstream): (NodeKeys, NodeKeys, NodeKeys),
) -> PyResult<Arc<Self>> {
let state = Arc::new(Self {
execution: worker.get().execution(worker.py())?,
worker: worker.clone().unbind(),
completed: Mutex::default(),
checks: Mutex::default(),
attached,
blocked,
upstream,
causes: Mutex::default(),
checked: Mutex::default(),
returned: AtomicBool::new(false),
refusal: Mutex::default(),
Expand Down Expand Up @@ -174,6 +190,7 @@ impl WorkerBuild {
Ok(())
}

/// A skip in a build names the failed node upstream of it; a test or audit run that raised skips what it never checked, with no cause.
fn complete(
&self,
py: Python<'_>,
Expand All @@ -195,6 +212,14 @@ impl WorkerBuild {
{
return Ok(());
}
let skipped_because_of = match outcome {
NodeOutcome::Skipped => match self.execution.skipped_nodes.get(key) {
Some(cause) => Some(cause.clone()),
None if !self.checks_only() => self.failed_upstream(key)?,
None => None,
},
_ => None,
};
self.worker.bind(py).get().change(
py,
&format!("node-finished/{key}"),
Expand All @@ -203,10 +228,20 @@ impl WorkerBuild {
outcome,
rows_affected,
error_code,
skipped_because_of: self.execution.skipped_nodes.get(key).cloned(),
skipped_because_of: skipped_because_of.clone(),
error_message: error_message.as_deref().map(redact_error_text),
},
)?;
let cause = match outcome {
NodeOutcome::Failed => Some(key.to_owned()),
_ => skipped_because_of,
};
if let Some(cause) = cause {
self.causes
.lock()
.map_err(|_| poisoned())?
.insert(key.into(), cause);
}
self.completed
.lock()
.map_err(|_| poisoned())?
Expand Down Expand Up @@ -331,8 +366,58 @@ impl WorkerBuild {
Ok(())
}

/// The failed node nearest upstream of `key` in this run, or the cause a skipped one carries.
fn failed_upstream(&self, key: &str) -> PyResult<Option<String>> {
let causes = self.causes.lock().map_err(|_| poisoned())?;
let mut seen: BTreeSet<&str> = BTreeSet::from([key]);
let mut pending: VecDeque<&str> = VecDeque::from([key]);
while let Some(node) = pending.pop_front() {
for upstream in self.upstream.get(node).into_iter().flatten() {
if let Some(cause) = causes.get(upstream) {
return Ok(Some(cause.clone()));
}
if seen.insert(upstream) {
pending.push_back(upstream);
}
}
}
Ok(None)
}

/// A failing SQL test, found by its compiled name such as `parent [case]`, fails the models it blocks with its error.
fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> {
let py = result.py();
let outcome: String = field(result, "outcome")?.str()?.extract()?;
if SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass)
|| self.worker.bind(py).get().refresh_cancellation(py)?
{
return Ok(());
}
let name: String = field(result, "test_name")?.extract()?;
let key = format!("{SQL_TEST_KEY_PREFIX}{name}");
let error: Option<String> = match result.getattr_opt("error_code")? {
Some(value) => value.extract()?,
None => None,
};
let message: Option<String> = match result.getattr_opt("error_message")? {
Some(value) => value.extract()?,
None => None,
};
for model in self.blocked.get(&key).into_iter().flatten() {
self.complete(
py,
model,
(NodeOutcome::Failed, None, (error.clone(), message.clone())),
)?;
}
Ok(())
}

/// Existing native progress callbacks publish selected-node evidence without Python relays.
pub(crate) fn completed(&self, result: &Bound<'_, PyAny>, key: &str) -> PyResult<()> {
if key.starts_with(SQL_TEST_KEY_PREFIX) {
return self.finished_test(result);
}
if self.execution.claim.request.selection.contains(key) {
let status: String = field(result, "status")?.str()?.extract()?;
let outcome = match status.as_str() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,14 @@ use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation;
use crate::bindings::_helpers::boundary::python_calls::KeywordArguments;
use crate::bindings::_helpers::compile_driver::pipeline::compiled_phase;
use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot;
use crate::bindings::_helpers::compiled_project::snapshot_execution::NodeKeys;
use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView;
use crate::bindings::_helpers::location_command::constants::{EXECUTE_ACTION, PUBLISH_ACTION};
use crate::bindings::_helpers::location_command::runtime::run_runtime;
use crate::bindings::_helpers::location_command::snapshot::{
build_orchestration_snapshot, orchestration_snapshot,
};
use crate::bindings::_helpers::location_command::worker::execute_worker;
use crate::bindings::_helpers::location_command::worker::{blocked_nodes, execute_worker};
use crate::bindings::_helpers::native_host::host::write_console;
use crate::bindings::_helpers::query_command::project::{
ProjectAdapter, ProjectInputs, project_adapter,
Expand Down Expand Up @@ -113,7 +114,7 @@ fn compile_location<'py>(
py: Python<'py>,
arguments: &KeywordArguments<'py>,
input: &SnapshotInput,
) -> PyResult<(NativeOrchestrationSnapshot, Bound<'py, PyAny>)> {
) -> PyResult<(NativeOrchestrationSnapshot, Bound<'py, PyAny>, NodeKeys)> {
status(py, "Discovering location project...")?;
let ProjectAdapter {
project_dir,
Expand Down Expand Up @@ -152,7 +153,7 @@ fn compile_location<'py>(
),
)?;
status(py, "Location snapshot compiled.")?;
Ok((snapshot, discovered))
Ok((snapshot, discovered, blocked_nodes(&project)?))
}

pub(super) fn location_arguments<'py>(
Expand Down Expand Up @@ -205,7 +206,7 @@ pub(super) fn publish_location(
.value("project_dir")?
.extract::<std::path::PathBuf>()?;
let input = metadata(py, arguments, &directory)?;
let (snapshot, _) = compile_location(py, arguments, &input)?;
let (snapshot, _, _) = compile_location(py, arguments, &input)?;
let locations: BTreeSet<_> = snapshot
.input
.nodes
Expand Down Expand Up @@ -287,10 +288,10 @@ pub(crate) fn run_location<'py>(
arguments.value("run_id")?.extract::<String>()?,
arguments.value("claim_id")?.extract::<String>()?,
);
let (snapshot, discovered) = compile_location(py, &arguments, &input)?;
let (snapshot, discovered, blocked) = compile_location(py, &arguments, &input)?;
return execute_worker(
py,
(&snapshot, discovered),
(&snapshot, discovered, blocked),
&arguments,
(&bearer, identity, recall),
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@ use crate::bindings::_helpers::command_runtime::warehouses::{
WarehouseInputs, configured_warehouses,
};
use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config;
use crate::bindings::_helpers::compiled_project::constants::SQL_TEST_KEY_PREFIX;
use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot;
use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild;
use crate::bindings::_helpers::compiled_project::snapshot_execution::{NodeKeys, WorkerBuild};
use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestrationWorker;
use crate::bindings::_helpers::compiled_views::project_reads::sql_tests;
use crate::bindings::_helpers::discovery::project_config::configured_changes;
use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command;
use crate::bindings::_helpers::location_command::constants::{
Expand Down Expand Up @@ -73,6 +75,38 @@ fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap<String, Ve
attached
}

/// Each compiled SQL test with the models a failure of it blocks in a build: its expected, assertion and reference targets, as the scheduler's `scope_deps` names them.
pub(super) fn blocked_nodes(project: &Bound<'_, PyAny>) -> PyResult<NodeKeys> {
let mut blocked: NodeKeys = BTreeMap::new();
for test in sql_tests(project)?.try_iter()? {
let test = test?;
let name: String = field(&field(&test, "key")?, "name")?.extract()?;
let models: Vec<String> = field(&test, "target_model_names")?.extract()?;
blocked.insert(
format!("{SQL_TEST_KEY_PREFIX}{name}"),
models
.into_iter()
.map(|model| format!("model:{model}"))
.collect(),
);
}
Ok(blocked)
}

/// Each snapshot node's direct upstream node keys.
fn upstream_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap<String, Vec<String>> {
snapshot
.input
.nodes
.iter()
.map(|(key, node)| {
let deps = node.definition.get("deps").and_then(Value::as_array);
let deps = deps.into_iter().flatten().filter_map(Value::as_str);
(key.clone(), deps.map(str::to_owned).collect())
})
.collect()
}

/// A job run's `--changed` settings: its whole selection, and source data counted unless configured.
fn job_run_settings(
project_dir: &Bound<'_, PyAny>,
Expand Down Expand Up @@ -257,7 +291,7 @@ fn run_cancellable<'py>(

pub(super) fn execute_worker<'py>(
py: Python<'py>,
(snapshot, discovered): (&NativeOrchestrationSnapshot, Bound<'py, PyAny>),
(snapshot, discovered, blocked): (&NativeOrchestrationSnapshot, Bound<'py, PyAny>, NodeKeys),
arguments: &KeywordArguments<'py>,
(bearer, identity, recall): (&str, (String, String), Option<&InvocationRecall>),
) -> PyResult<i32> {
Expand All @@ -272,7 +306,10 @@ pub(super) fn execute_worker<'py>(
)
.map_err(|error| PyValueError::new_err(error.to_string()))?,
)?;
let worker = WorkerBuild::new(&owner, attached_nodes(snapshot))?;
let worker = WorkerBuild::new(
&owner,
(attached_nodes(snapshot), blocked, upstream_nodes(snapshot)),
)?;
if let Some(recall) = recall {
*recall
.worker
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,10 @@ class WorkerExecutionTestCase:
concurrency_limits: dict[str, int] = field(default_factory=dict)
launcher: Literal["native", "uncooperative", "docker"] = "native"
warehouse_setup: Callable[[Path], None] = lambda _: None
# Each run node as (key, status, error code, skipped because of, error message).
expected_run_nodes: list[tuple[str, str, str | None, str | None, str | None]] = field(
default_factory=list
)


@dataclass(frozen=True)
Expand Down
Loading
Loading