diff --git a/Makefile b/Makefile index f3e8c2144c..0a7d58ea32 100644 --- a/Makefile +++ b/Makefile @@ -167,6 +167,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_terminal_failure.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_load_sources.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_storage_key.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_source_end_audits.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_check_outcomes.py \ diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs index 092e82a5c7..b6075400ab 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs @@ -6,6 +6,7 @@ use crate::dag_artifact::_helpers::nodes::{ loader_or_source_id, node_id, python_node_id, sources_by_loader, }; use crate::dag_artifact::_helpers::values::{object, text}; +use crate::dag_artifact::constants::LOADER_KIND; use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; use crate::dag_artifact::types::DagValue; use crate::graph::models::ProjectGraph; @@ -19,21 +20,28 @@ pub(crate) fn python_kinds(python: &DagPythonGraphInput) -> HashMap<&str, &str> .collect() } -fn python_edges(python: &DagPythonGraphInput) -> Vec<(String, String)> { +/// Python dependency and SQL read edges; a loader a source declares is drawn as that source. +fn python_edges( + python: &DagPythonGraphInput, + sources: &HashMap<&str, &str>, +) -> Vec<(String, String)> { let kinds: HashMap<&str, &str> = python_kinds(python); + let id = |name: &str| { + let kind: &str = kinds[name]; + if kind == LOADER_KIND { + loader_or_source_id(name, sources) + } else { + python_node_id(kind, name) + } + }; let mut edges: Vec<(String, String)> = python .dependency_edges .iter() - .map(|(upstream, downstream)| { - ( - python_node_id(kinds[upstream.as_str()], upstream), - python_node_id(kinds[downstream.as_str()], downstream), - ) - }) + .map(|(upstream, downstream)| (id(upstream), id(downstream))) .collect(); for node in &python.nodes { for dependency in &node.sql_deps { - edges.push((node_id(dependency), python_node_id(&node.kind, &node.name))); + edges.push((node_id(dependency), id(&node.name))); } } edges @@ -61,7 +69,7 @@ pub(crate) fn edges( } } if let Some(python) = python { - edges.extend(python_edges(python)); + edges.extend(python_edges(python, &sources)); } edges.sort(); DagValue::Array( diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs index a6ab1a2e78..73c51d7f8d 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs @@ -4,7 +4,7 @@ use crate::dag_artifact::main::build_dag_json::build_dag_json; use crate::dag_artifact::models::{ DagColumnInput, DagLoaderInput, DagPythonNodeInput, DagSourceEntryInput, DagSourceInput, }; -use crate::dag_artifact::tests::test_types::BuildDagJsonTestCase; +use crate::dag_artifact::tests::test_types::{BuildDagJsonTestCase, DagEdgeTestCase}; use crate::dag_artifact::types::DagValue; use crate::graph::models::ProjectGraph; use crate::python_values::main::text_key::text_key; @@ -89,6 +89,28 @@ pub(crate) fn built(test_case: &BuildDagJsonTestCase) -> Result Vec<(String, String)> { + let artifact: String = build_dag_json( + &ProjectGraph::default(), + &test_case.project, + Some(&test_case.python), + "orders_project", + ) + .expect("artifact"); + let lines: Vec<&str> = artifact.lines().map(str::trim).collect(); + lines + .windows(2) + .filter_map(|pair| { + let from: &str = pair[0] + .strip_prefix("\"from_id\": \"")? + .strip_suffix("\",")?; + let to: &str = pair[1].strip_prefix("\"to_id\": \"")?.strip_suffix('"')?; + Some((from.to_owned(), to.to_owned())) + }) + .collect() +} + /// Source meta with a `None` owner and the given team. pub(crate) fn team_meta(value: DagValue) -> DagValue { DagValue::Object(vec![ diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs index 1c540c5ae9..4970e8e653 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_build_dag_json.rs @@ -1,7 +1,9 @@ use crate::dag_artifact::errors::DagArtifactError; use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; -use crate::dag_artifact::tests::helpers::{built, key, loader, python_node, source, team_meta}; -use crate::dag_artifact::tests::test_types::BuildDagJsonTestCase; +use crate::dag_artifact::tests::helpers::{ + built, dag_edges, key, loader, python_node, source, team_meta, +}; +use crate::dag_artifact::tests::test_types::{BuildDagJsonTestCase, DagEdgeTestCase}; use crate::dag_artifact::types::DagValue; const EMPTY_JSON: &str = "{ @@ -172,3 +174,46 @@ fn given_project_when_building_dag_json_then_python_artifact_agrees() { ); } } + +#[test] +fn given_a_loader_depending_on_a_task_when_building_dag_json_then_the_edge_reaches_its_source() { + let test_cases = [DagEdgeTestCase { + description: "a task upstream of a declared loader points at the loader's source", + project: DagProjectInput { + sources: vec![source( + "partner_scores", + Some("partner_scores"), + DagValue::Object(Vec::new()), + )], + loaders: vec![loader("partner_scores", None, &[])], + ..DagProjectInput::default() + }, + python: DagPythonGraphInput { + nodes: vec![ + python_node("task", "score_orders", vec![key("model", "orders")]), + python_node("loader", "partner_scores", Vec::new()), + ], + dependency_edges: vec![("score_orders".to_owned(), "partner_scores".to_owned())], + kinds_by_name: vec![ + ("score_orders".to_owned(), "task".to_owned()), + ("partner_scores".to_owned(), "loader".to_owned()), + ], + }, + expected_edges: &[ + ("model:orders", "task:score_orders"), + ("task:score_orders", "source:partner_scores"), + ], + }]; + for test_case in test_cases { + assert_eq!( + dag_edges(&test_case), + test_case + .expected_edges + .iter() + .map(|(from, to)| ((*from).to_owned(), (*to).to_owned())) + .collect::>(), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs index 5b05ed6281..6d88a2b5a6 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs @@ -28,3 +28,11 @@ pub(crate) struct SpecKindsTestCase { pub(crate) materialization_type: &'static str, pub(crate) expected_kinds: Vec<&'static str>, } + +pub(crate) struct DagEdgeTestCase { + pub(crate) description: &'static str, + pub(crate) project: DagProjectInput, + pub(crate) python: DagPythonGraphInput, + /// `(from_id, to_id)` of every edge, sorted. + pub(crate) expected_edges: &'static [(&'static str, &'static str)], +} diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs b/crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs index 9ff9af6d87..5500c932ef 100644 --- a/crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs +++ b/crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs @@ -1,22 +1,24 @@ //! Python's planner selector grammar and resolution over the project graph. -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashSet}; +use std::convert::Infallible; use crate::assembly::project::types::ObjectKey; use crate::graph::constants::{ EMPTY_SELECTOR_CODE, EMPTY_VALUE_CODE, EXPANSION_MARKER, FOLDER_SEPARATOR, INTERSECTION_SEPARATOR, JOB_KIND, KIND_SEPARATOR, MISPLACED_MARKER_CODE, MISSING_NAME_CODE, - MODEL_RESOURCE, MODEL_ROOT, MODEL_ROOT_PREFIX, NAME_KIND, PATH_KIND, PATH_ROOT_CODE, - PATH_ROOT_ERROR, PATH_SELECTOR_CODE, PATH_SEPARATOR, PATTERN_CHARACTERS, PLANNER_DEFAULT_CODE, - SEED_RESOURCE, SOURCE_RESOURCE, SQL_FILE_SUFFIX, SUGGESTION_CUTOFF, SUGGESTION_LIMIT, - TABLE_FUNCTION_RESOURCE, TAG_KIND, TEST_KIND, UDF_RESOURCE, UNIT_TEST_CODE, - UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_KIND_CODE, UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, - UNKNOWN_TAG_CODE, UNMAPPED_KIND_CODE, UNMAPPED_KINDS, + MODEL_RESOURCE, MODEL_ROOT, MODEL_ROOT_PREFIX, NAME_KIND, NODE_KIND_KIND, PATH_KIND, + PATH_ROOT_CODE, PATH_ROOT_ERROR, PATH_SELECTOR_CODE, PATH_SEPARATOR, PATTERN_CHARACTERS, + PLANNER_DEFAULT_CODE, SEED_RESOURCE, SELECTABLE_NODE_KINDS, SOURCE_RESOURCE, SQL_FILE_SUFFIX, + SUGGESTION_CUTOFF, SUGGESTION_LIMIT, TABLE_FUNCTION_RESOURCE, TAG_KIND, TEST_KIND, + UDF_RESOURCE, UNIT_TEST_CODE, UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_KIND_CODE, + UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, UNMAPPED_KIND_CODE, UNMAPPED_KINDS, }; use crate::graph::errors::SelectorError; use crate::graph::models::{ BuildResources, EdgeDirection as Direction, ParsedSelector, ProjectGraph, }; +use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; use sqlbuild_core::text::main::close_matches::close_matches; use sqlbuild_core::text::main::glob_match::glob_match; @@ -36,12 +38,12 @@ pub(crate) fn resolve( select: &[String], exclude: &[String], ) -> Result, SelectorError> { - let all: Keys = graph.names.iter().map(|(_, key)| key.clone()).collect(); + let all = || -> Keys { graph.names.iter().map(|(_, key)| key.clone()).collect() }; if select.is_empty() && exclude.is_empty() { - return Ok(all.into_iter().collect()); + return Ok(all().into_iter().collect()); } let selected: Keys = if select.is_empty() { - all + all() } else { tokens(graph, select)? }; @@ -52,21 +54,27 @@ pub(crate) fn resolve( .collect()) } -/// Python's `expand_required_build_resources`: functions and seeds a selected scope needs. +/// Python's `expand_required_build_resources`: functions and seeds a selected scope needs; one walk from every selected key reaches the union of their upstream closures. pub(crate) fn build_resources( graph: &ProjectGraph, selected: &Keys, include: BuildResources, ) -> Keys { let mut expanded: Keys = selected.clone(); - for key in selected { - for upstream in graph.closure(key, Direction::Upstream) { - let wanted: bool = (include.upstream_functions && is_function(&upstream)) - || (include.upstream_seeds && upstream.0 == SEED_RESOURCE); - if wanted { - expanded.insert(upstream); - } - } + if include.upstream_functions || include.upstream_seeds { + let mut reached: HashSet<&ObjectKey> = HashSet::new(); + let Ok(()) = breadth_first_reach(selected, None, &mut reached, |current| { + Ok::<_, Infallible>(graph.edges(current, Direction::Upstream)) + }); + expanded.extend( + reached + .into_iter() + .filter(|upstream| { + (include.upstream_functions && is_function(upstream)) + || (include.upstream_seeds && upstream.0 == SEED_RESOURCE) + }) + .cloned(), + ); } if include.downstream_functions { for key in selected.iter().filter(|key| key.0 == MODEL_RESOURCE) { @@ -273,6 +281,7 @@ fn is_selector_kind(prefix: &str) -> bool { PATH_KIND, TEST_KIND, JOB_KIND, + NODE_KIND_KIND, ] .contains(&prefix) || UNMAPPED_KINDS.contains(&prefix) @@ -293,6 +302,7 @@ pub(crate) fn matched(graph: &ProjectGraph, kind: &str, value: &str) -> Resolved } PATH_KIND => folder(graph, value), JOB_KIND => job(graph, value), + NODE_KIND_KIND => of_kind(graph, value), TEST_KIND => Err(error( UNIT_TEST_CODE, format!( @@ -373,6 +383,49 @@ fn lookup(graph: &ProjectGraph, kind: &str, value: &str) -> Resolved { .collect()) } +/// The SQL resources of one node kind; Python node kinds need the Python node graph. +fn of_kind(graph: &ProjectGraph, value: &str) -> Resolved { + if !SELECTABLE_NODE_KINDS.contains(&value) { + return Err(unknown_node_kind(value)); + } + if UNMAPPED_KINDS.contains(&value) { + return Err(error( + UNMAPPED_KIND_CODE, + format!( + "selector '{NODE_KIND_KIND}{KIND_SEPARATOR}{value}' selects Python nodes; \ + they cannot be selected here" + ), + )); + } + let keys: Keys = graph + .names + .iter() + .filter(|(_, key)| key.0 == value) + .map(|(_, key)| key.clone()) + .collect(); + if keys.is_empty() { + return Err(error( + PLANNER_DEFAULT_CODE, + format!("no nodes found of kind '{value}'"), + )); + } + Ok(keys) +} + +/// `value` is not a node kind; suggest the closest ones. +pub(crate) fn unknown_node_kind(value: &str) -> SelectorError { + SelectorError { + code: UNKNOWN_NAME_CODE, + message: format!("unknown node kind '{value}'"), + help: suggestion(value, SELECTABLE_NODE_KINDS.to_vec()).or_else(|| { + Some(format!( + "node kinds are {}", + SELECTABLE_NODE_KINDS.join(", ") + )) + }), + } +} + /// The nodes the jobs and job groups `value` names or matches select. fn job(graph: &ProjectGraph, value: &str) -> Resolved { let Some(jobs) = &graph.jobs else { diff --git a/crates/sqlbuild-analysis/src/graph/constants.rs b/crates/sqlbuild-analysis/src/graph/constants.rs index 19d224cb54..f3896c96ae 100644 --- a/crates/sqlbuild-analysis/src/graph/constants.rs +++ b/crates/sqlbuild-analysis/src/graph/constants.rs @@ -19,6 +19,11 @@ pub(crate) const TAG_KIND: &str = "tag"; pub(crate) const PATH_KIND: &str = "path"; pub(crate) const TEST_KIND: &str = "test"; pub(crate) const JOB_KIND: &str = "job"; +pub(crate) const NODE_KIND_KIND: &str = "kind"; +/// The node kinds `kind:` selects, as the catalog names them: SQL resources, then Python nodes. +pub(crate) const SELECTABLE_NODE_KINDS: [&str; 9] = [ + "model", "source", "seed", "udf", "table_fn", "task", "asset", "loader", "check", +]; /// Selector kinds Python parses but `sqb compile` cannot map to a compiled resource. pub(crate) const UNMAPPED_KINDS: [&str; 4] = ["task", "asset", "loader", "check"]; pub(crate) const SUGGESTION_CUTOFF: f64 = 0.8; @@ -35,7 +40,7 @@ pub(crate) const UNKNOWN_PATH_CODE: &str = "S009"; pub(crate) const UNMAPPED_KIND_CODE: &str = "S010"; pub(crate) const PATH_ROOT_CODE: &str = "S012"; pub(crate) const UNIT_TEST_CODE: &str = "S013"; -pub(crate) const PLANNER_DEFAULT_CODE: &str = "S000"; +pub const PLANNER_DEFAULT_CODE: &str = "S000"; pub(crate) const PATH_ROOT_ERROR: &str = "path selectors require an explicit root: use 'models/' or 'python/'"; pub(crate) const UNIT_TEST_ONLY_TEST_AND_BUILD: &str = diff --git a/crates/sqlbuild-analysis/src/graph/main/named_selector_atoms.rs b/crates/sqlbuild-analysis/src/graph/main/named_selector_atoms.rs index 2565f78b95..f9f8bb5c51 100644 --- a/crates/sqlbuild-analysis/src/graph/main/named_selector_atoms.rs +++ b/crates/sqlbuild-analysis/src/graph/main/named_selector_atoms.rs @@ -1,7 +1,9 @@ -use crate::graph::constants::{INTERSECTION_SEPARATOR, JOB_KIND, PATH_KIND, TAG_KIND}; +use crate::graph::constants::{ + INTERSECTION_SEPARATOR, JOB_KIND, NODE_KIND_KIND, PATH_KIND, TAG_KIND, +}; use crate::graph::models::ParsedSelector; -/// The atoms of `selection` that name one node: bare or typed names, without `+`, globs, tags, paths or jobs. +/// The atoms of `selection` that name one node: bare or typed names, without `+`, globs, tags, paths, jobs or kinds. #[must_use] pub fn named_selector_atoms(selection: &str) -> Vec { selection @@ -20,7 +22,7 @@ fn names_one_node(atom: &str) -> bool { upstream: false, downstream: false, }) => { - ![TAG_KIND, PATH_KIND, JOB_KIND].contains(&kind.as_str()) + ![TAG_KIND, PATH_KIND, JOB_KIND, NODE_KIND_KIND].contains(&kind.as_str()) && !crate::graph::_helpers::selectors::is_pattern(&value) } _ => false, diff --git a/crates/sqlbuild-analysis/src/graph/mod.rs b/crates/sqlbuild-analysis/src/graph/mod.rs index 1bb62c4b82..279f59501e 100644 --- a/crates/sqlbuild-analysis/src/graph/mod.rs +++ b/crates/sqlbuild-analysis/src/graph/mod.rs @@ -1,7 +1,7 @@ //! The compiled project graph: dependency indexes and selector resolution. pub(crate) mod _helpers; -pub(crate) mod constants; +pub mod constants; pub mod errors; pub mod main; pub mod models; diff --git a/crates/sqlbuild-analysis/src/graph/tests/test_named_selector_atoms.rs b/crates/sqlbuild-analysis/src/graph/tests/test_named_selector_atoms.rs index d176cccfd0..20c0594130 100644 --- a/crates/sqlbuild-analysis/src/graph/tests/test_named_selector_atoms.rs +++ b/crates/sqlbuild-analysis/src/graph/tests/test_named_selector_atoms.rs @@ -29,6 +29,11 @@ fn given_a_selection_when_listing_named_atoms_then_only_single_node_names_count( selection: "tag:orders tag:finance", expected_atoms: &[], }, + NamedAtomsTestCase { + description: "kinds do not count", + selection: "kind:model kind:task", + expected_atoms: &[], + }, NamedAtomsTestCase { description: "folder paths do not count", selection: "models/orders python/orders path:models/finance", diff --git a/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs b/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs index 690b589d4a..e89b991aea 100644 --- a/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs +++ b/crates/sqlbuild-analysis/src/graph/tests/test_selector_scenarios.rs @@ -206,6 +206,20 @@ fn given_selectors_when_resolving_then_selects_expected_names() { "raw_orders", ], }, + SelectedNamesTestCase { + description: "kind selects every resource of one kind", + project: DIAMOND_PROJECT, + select: &["kind:source"], + exclude: &[], + expected_names: &["raw_customers", "raw_orders"], + }, + SelectedNamesTestCase { + description: "+ expands its own comma part before the kind narrows it", + project: DIAMOND_PROJECT, + select: &["+joined,kind:source"], + exclude: &[], + expected_names: &["raw_customers", "raw_orders"], + }, SelectedNamesTestCase { description: "downstream expansion", project: DIAMOND_PROJECT, @@ -512,6 +526,26 @@ fn given_unresolvable_selectors_when_resolving_then_fails_with_message() { select: &["nonexistent_model"], expected_error: ("S007", "unknown selector name 'nonexistent_model'", None), }, + ProjectSelectorErrorTestCase { + description: "unknown node kind suggests the closest", + project: DIAMOND_PROJECT, + select: &["kind:modle"], + expected_error: ( + "S007", + "unknown node kind 'modle'", + Some("did you mean 'model'?"), + ), + }, + ProjectSelectorErrorTestCase { + description: "a Python node kind needs the Python node graph", + project: DIAMOND_PROJECT, + select: &["kind:task"], + expected_error: ( + "S010", + "selector 'kind:task' selects Python nodes; they cannot be selected here", + None, + ), + }, ProjectSelectorErrorTestCase { description: "name glob matching nothing", project: DIAMOND_PROJECT, diff --git a/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs b/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs index 94380de640..6d8da0ca03 100644 --- a/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs +++ b/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs @@ -2,11 +2,12 @@ use std::collections::{BTreeSet, HashMap}; +use crate::assembly::project::types::ObjectKey; use crate::python_plan::main::upstream_closure::upstream_closure; -use crate::python_plan::models::{LifecycleNode, RunLifecycle}; +use crate::python_plan::models::{LifecycleNode, RoundTripLoader, RunLifecycle}; use crate::python_selection::constants::{ASSET_KIND, LOADER_KIND, TASK_KIND}; -/// Ingress is every selected loader and its selected upstream tasks, assets and loaders. +/// Ingress is each selected loader whose selected upstream reads no SQL, with that upstream; the rest load after the SQL. #[must_use] pub fn run_lifecycle( selected: &BTreeSet, @@ -14,17 +15,32 @@ pub fn run_lifecycle( upstream: &HashMap>, ) -> RunLifecycle { let kind = |name: &str| nodes.get(name).map_or("", |node| node.kind.as_str()); - let loaders: BTreeSet = selected - .iter() - .filter(|name| { - nodes - .get(name.as_str()) - .is_some_and(|node| node.kind == LOADER_KIND) - }) - .cloned() - .collect(); - let mut ingress: BTreeSet = loaders.clone(); - for loader in &loaders { + let runs = |name: &str| [TASK_KIND, ASSET_KIND].contains(&kind(name)); + let mut ingress_loaders: BTreeSet = BTreeSet::new(); + let mut round_trip_loaders: Vec = Vec::new(); + for loader in selected.iter().filter(|name| kind(name) == LOADER_KIND) { + let closure: Vec = upstream_closure(loader, upstream) + .into_iter() + .filter(|name| selected.contains(name)) + .collect(); + let waits_for: BTreeSet = closure + .iter() + .filter(|name| runs(name)) + .filter_map(|name| nodes.get(name)) + .flat_map(|node| node.sql_deps.iter().cloned()) + .collect(); + if waits_for.is_empty() { + ingress_loaders.insert(loader.clone()); + continue; + } + round_trip_loaders.push(RoundTripLoader { + name: loader.clone(), + waits_for: waits_for.into_iter().collect(), + gates: closure.into_iter().filter(|name| runs(name)).collect(), + }); + } + let mut ingress: BTreeSet = ingress_loaders.clone(); + for loader in &ingress_loaders { for name in upstream_closure(loader, upstream) { if selected.contains(&name) && [TASK_KIND, ASSET_KIND, LOADER_KIND].contains(&kind(&name)) @@ -35,12 +51,13 @@ pub fn run_lifecycle( } let read_side: Vec = selected .iter() - .filter(|name| !ingress.contains(*name) && [TASK_KIND, ASSET_KIND].contains(&kind(name))) + .filter(|name| !ingress.contains(*name) && runs(name)) .cloned() .collect(); RunLifecycle { ingress_names: ingress.into_iter().collect(), - ingress_loader_names: loaders.into_iter().collect(), + ingress_loader_names: ingress_loaders.into_iter().collect(), read_side_names: read_side, + round_trip_loaders, } } diff --git a/crates/sqlbuild-analysis/src/python_plan/models.rs b/crates/sqlbuild-analysis/src/python_plan/models.rs index 9d095763a8..b04f58cbd1 100644 --- a/crates/sqlbuild-analysis/src/python_plan/models.rs +++ b/crates/sqlbuild-analysis/src/python_plan/models.rs @@ -1,10 +1,24 @@ //! Run lifecycle inputs and the regions selected Python nodes run in. +use crate::assembly::project::types::ObjectKey; + /// One Python node as lifecycle classification reads it. #[derive(Clone, Debug, PartialEq, Eq)] pub struct LifecycleNode { /// `task`, `asset`, `loader` or `check`. pub kind: String, + /// The SQL resources the node reads, in authored order. + pub sql_deps: Vec, +} + +/// A selected loader whose selected Python upstream reads SQL, so it runs after that SQL is built. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RoundTripLoader { + pub name: String, + /// The SQL its selected upstream tasks and assets read, which its load waits for; sorted. + pub waits_for: Vec, + /// Its selected upstream tasks and assets, which must succeed for it to load; sorted. + pub gates: Vec, } /// Selected Python nodes split into ingress and read-side regions, each sorted. @@ -13,4 +27,6 @@ pub struct RunLifecycle { pub ingress_names: Vec, pub ingress_loader_names: Vec, pub read_side_names: Vec, + /// Loaders that load in the SQL build after the SQL their upstream reads, sorted by name. + pub round_trip_loaders: Vec, } diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs index f26e0d968c..873f53de24 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/helpers.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use crate::python_plan::models::LifecycleNode; +use crate::python_plan::tests::test_types::LifecycleNodeSpec; pub(crate) fn edges(rows: &[(&str, &[&str])]) -> HashMap> { rows.iter() @@ -13,15 +14,24 @@ pub(crate) fn edges(rows: &[(&str, &[&str])]) -> HashMap> { .collect() } -pub(crate) fn lifecycle_nodes(rows: &[(&str, &str)]) -> HashMap { +pub(crate) fn lifecycle_nodes(rows: &[LifecycleNodeSpec]) -> HashMap { rows.iter() - .map(|(name, kind)| { + .map(|(name, kind, sql_deps)| { ( (*name).to_owned(), LifecycleNode { kind: (*kind).to_owned(), + sql_deps: sql_deps + .iter() + .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) + .collect(), }, ) }) .collect() } + +/// Owned copies of `values`. +pub(crate) fn names(values: &[&str]) -> Vec { + values.iter().map(|name| (*name).to_owned()).collect() +} diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs index f37f49ba39..69d0a9d4c9 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs @@ -1,7 +1,8 @@ use std::collections::BTreeSet; use crate::python_plan::main::run_lifecycle::run_lifecycle; -use crate::python_plan::tests::helpers::{edges, lifecycle_nodes}; +use crate::python_plan::models::{RoundTripLoader, RunLifecycle}; +use crate::python_plan::tests::helpers::{edges, lifecycle_nodes, names}; use crate::python_plan::tests::test_types::LifecycleTestCase; #[test] @@ -11,24 +12,63 @@ fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { description: "a loader pulls its selected upstream tasks into ingress", selected: &["fetch_rates", "load_orders", "report", "check_orders"], nodes: &[ - ("fetch_rates", "task"), - ("load_orders", "loader"), - ("report", "asset"), - ("check_orders", "check"), + ("fetch_rates", "task", &[]), + ("load_orders", "loader", &[]), + ("report", "asset", &[("model", "orders")]), + ("check_orders", "check", &[]), ], upstream: &[("load_orders", &["fetch_rates", "unselected"])], expected_ingress: &["fetch_rates", "load_orders"], expected_loaders: &["load_orders"], expected_read_side: &["report"], + expected_round_trips: &[], }, LifecycleTestCase { description: "a selected loader upstream of another loader is ingress", selected: &["sync_orders", "load_orders"], - nodes: &[("sync_orders", "loader"), ("load_orders", "loader")], + nodes: &[ + ("sync_orders", "loader", &[]), + ("load_orders", "loader", &[]), + ], upstream: &[("load_orders", &["sync_orders"])], expected_ingress: &["load_orders", "sync_orders"], expected_loaders: &["load_orders", "sync_orders"], expected_read_side: &[], + expected_round_trips: &[], + }, + LifecycleTestCase { + description: "a loader whose selected upstream reads SQL loads after it, gated by it", + selected: &["score_orders", "package_scores", "partner_scores"], + nodes: &[ + ("score_orders", "task", &[("model", "orders")]), + ("package_scores", "asset", &[]), + ("partner_scores", "loader", &[]), + ], + upstream: &[ + ("partner_scores", &["package_scores"]), + ("package_scores", &["score_orders"]), + ], + expected_ingress: &[], + expected_loaders: &[], + expected_read_side: &["package_scores", "score_orders"], + expected_round_trips: &[( + "partner_scores", + &[("model", "orders")], + &["package_scores", "score_orders"], + )], + }, + LifecycleTestCase { + description: "a loader selected without its SQL-reading upstream is ordinary ingress", + selected: &["partner_scores"], + nodes: &[ + ("score_orders", "task", &[("model", "orders")]), + ("partner_scores", "loader", &[]), + ], + upstream: &[("partner_scores", &["score_orders"])], + expected_ingress: &["partner_scores"], + expected_loaders: &["partner_scores"], + expected_read_side: &[], + expected_round_trips: &[], }, ]; @@ -39,35 +79,31 @@ fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { .map(|name| (*name).to_owned()) .collect(); - let lifecycle = run_lifecycle( + let lifecycle: RunLifecycle = run_lifecycle( &selected, &lifecycle_nodes(test_case.nodes), &edges(test_case.upstream), ); assert_eq!( - ( - lifecycle.ingress_names, - lifecycle.ingress_loader_names, - lifecycle.read_side_names, - ), - ( - test_case - .expected_ingress - .iter() - .map(|name| (*name).to_owned()) - .collect::>(), - test_case - .expected_loaders - .iter() - .map(|name| (*name).to_owned()) - .collect::>(), - test_case - .expected_read_side + lifecycle, + RunLifecycle { + ingress_names: names(test_case.expected_ingress), + ingress_loader_names: names(test_case.expected_loaders), + read_side_names: names(test_case.expected_read_side), + round_trip_loaders: test_case + .expected_round_trips .iter() - .map(|name| (*name).to_owned()) - .collect::>(), - ), + .map(|(name, waits_for, gates)| RoundTripLoader { + name: (*name).to_owned(), + waits_for: waits_for + .iter() + .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) + .collect(), + gates: names(gates), + }) + .collect(), + }, "{}", test_case.description ); diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs index 91e2c9879c..c4044b7ecc 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs @@ -15,10 +15,25 @@ pub(crate) struct ClosureTestCase { pub(crate) struct LifecycleTestCase { pub(crate) description: &'static str, pub(crate) selected: &'static [&'static str], - /// `(name, kind)`. - pub(crate) nodes: &'static [(&'static str, &'static str)], + pub(crate) nodes: &'static [LifecycleNodeSpec], pub(crate) upstream: &'static [(&'static str, &'static [&'static str])], pub(crate) expected_ingress: &'static [&'static str], pub(crate) expected_loaders: &'static [&'static str], pub(crate) expected_read_side: &'static [&'static str], + /// `(loader, SQL it waits for, tasks and assets that gate it)`. + pub(crate) expected_round_trips: &'static [ExpectedRoundTrip], } + +/// `(name, kind, SQL reads)`. +pub(crate) type LifecycleNodeSpec = ( + &'static str, + &'static str, + &'static [(&'static str, &'static str)], +); + +/// `(loader, SQL it waits for, tasks and assets that gate it)`. +pub(crate) type ExpectedRoundTrip = ( + &'static str, + &'static [(&'static str, &'static str)], + &'static [&'static str], +); diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs index 7fc0e9cd4a..7e9fe9a25d 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs @@ -12,10 +12,10 @@ use crate::graph::main::parse_selector::parse_selector; use crate::graph::models::{EdgeDirection, ParsedSelector, ProjectGraph}; use crate::python_selection::constants::{ ASSET_KIND, CHECK_KIND, INTERSECTION_SEPARATOR, JOB_KIND, LOADER_KIND, NAME_KIND, - PATH_BETWEEN_CODE, PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PATTERN_CHARACTERS, - SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, UNIT_TEST_CODE, - UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, - UNMAPPED_KIND_CODE, + NODE_KIND_KIND, PATH_BETWEEN_CODE, PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, + PATTERN_CHARACTERS, PLANNER_DEFAULT_CODE, SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, + UNIT_TEST_CODE, UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, + UNKNOWN_TAG_CODE, UNMAPPED_KIND_CODE, }; use crate::python_selection::models::{ NodeIndex, PythonSelectionGraph, PythonSelectionNode, SelectionContext, SqlSelectionGraph, @@ -181,6 +181,7 @@ fn single(index: &NodeIndex<'_>, raw: &str) -> Resolved { TEST_KIND => return Err(unit_test_error(raw)), TAG_KIND => tagged(index, &value)?, PATH_KIND => under_path(index, &value)?, + NODE_KIND_KIND => of_kind(index, &value)?, _ => { let names: Names = lookup(index, &kind, &value)?; if names.is_empty() { @@ -213,6 +214,29 @@ fn tagged(index: &NodeIndex<'_>, tag: &str) -> Resolved { Ok(names) } +/// The Python nodes of one kind, such as every `task`. +fn of_kind(index: &NodeIndex<'_>, kind: &str) -> Resolved { + if ![TASK_KIND, ASSET_KIND, LOADER_KIND, CHECK_KIND].contains(&kind) { + return Err(error( + UNMAPPED_KIND_CODE, + format!("node kind '{kind}' is not a Python node kind"), + )); + } + let names: Names = index + .by_name + .values() + .filter(|node| node.kind == kind) + .map(|node| node.name.clone()) + .collect(); + if names.is_empty() { + return Err(error( + PLANNER_DEFAULT_CODE, + format!("no nodes found of kind '{kind}'"), + )); + } + Ok(names) +} + fn under_path(index: &NodeIndex<'_>, value: &str) -> Resolved { let folder: String = selector_folder(value); let root: &str = folder_root(&folder); diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index 56bf52afce..c8c7a6c2a4 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -22,8 +22,8 @@ use crate::python_selection::_helpers::validation::{ }; use crate::python_selection::constants::{ ASSET_KIND, CHECK_KIND, EXPANSION_MARKER, INTERSECTION_SEPARATOR, JOB_KIND, LOADER_KIND, - NAME_KIND, PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PLANNER_DEFAULT_CODE, SEED_KIND, - SOURCE_KIND, SOURCE_RESOURCE, SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, + NAME_KIND, NODE_KIND_KIND, PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PLANNER_DEFAULT_CODE, + SEED_KIND, SOURCE_KIND, SOURCE_RESOURCE, SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, UNKNOWN_NAME_CODE, UNKNOWN_TAG_CODE, }; use crate::python_selection::models::{Atom, PythonSelectionNode, RunSelection, SelectionContext}; @@ -348,6 +348,12 @@ fn single_side(context: &SelectionContext<'_>, raw: &str) -> Resolved { return resolve_typed(context, raw, (&kind, &value)); } TAG_KIND => resolve_tag(context, raw, &value)?, + NODE_KIND_KIND + if [TASK_KIND, ASSET_KIND, LOADER_KIND, CHECK_KIND].contains(&value.as_str()) => + { + resolve_python(context, raw)? + } + NODE_KIND_KIND => resolve_sql(context, raw)?, NAME_KIND => resolve_name(context, raw, &value)?, _ => { return Err(error( diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs index 9bed1e9624..a87a6a252a 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs @@ -1,6 +1,10 @@ //! Python's SQL/Python graph boundary rules and selected-dependency checks. -use std::collections::{BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; + +use sqlbuild_core::graph_walk::main::find_cycle::find_cycle as find_dependency_cycle; + +use crate::assembly::project::types::ObjectKey; use crate::graph::errors::SelectorError; use crate::graph::models::EdgeDirection; @@ -127,34 +131,180 @@ fn validate_python_sql_refs(context: &SelectionContext<'_>) -> Result<(), Select Ok(()) } -fn validate_loader_upstreams(context: &SelectionContext<'_>) -> Result<(), SelectorError> { - for loader in context +/// The dependency cycle through a loader, which every cycle across SQL and Python passes. +fn dependency_cycle(context: &SelectionContext<'_>) -> Option { + let cycle: Vec = round_trip_cycle(context, &loader_names(context))?; + Some(SelectorError { + code: PLANNER_DEFAULT_CODE, + message: format!( + "{} is in a dependency cycle: {}", + cycle[0], + cycle.join(" -> ") + ), + help: Some( + "each arrow points to what a node depends on; a task or asset upstream of a \ + loader cannot read anything built from the source that loader fills" + .to_owned(), + ), + }) +} + +fn loader_names<'graph>(context: &SelectionContext<'graph>) -> Vec<&'graph str> { + context .nodes .graph .nodes .iter() .filter(|node| node.kind == LOADER_KIND) - { - for upstream_name in context + .map(|node| node.name.as_str()) + .collect() +} + +fn validate_loader_upstreams(context: &SelectionContext<'_>) -> Result<(), SelectorError> { + if let Some(cycle) = dependency_cycle(context) { + return Err(cycle); + } + let loaders: Vec<&str> = loader_names(context); + if context.sql.allow_warehouse_round_trips { + return Ok(()); + } + let terminal: HashMap<&str, &str> = terminal_loaders(context.sql); + for loader in loaders { + let Some(path) = sql_reading_upstream(context, loader) else { + continue; + }; + let reader: &str = path[path.len() - 1]; + let kind: &str = context .nodes - .closure([loader.name.as_str()], EdgeDirection::Upstream) + .node(reader) + .map_or(TASK_KIND, |node| &node.kind); + let through: String = if path.len() > 1 { + let names: Vec = path[..path.len() - 1] + .iter() + .map(|name| python_label(context, name)) + .collect(); + format!(" (through {})", names.join(", ")) + } else { + String::new() + }; + let source: &str = terminal.get(loader).copied().unwrap_or(loader); + return Err(SelectorError { + code: PLANNER_DEFAULT_CODE, + message: format!( + "loader '{loader}' depends on {kind} '{reader}'{through}, which reads SQL\n \ + This is a warehouse round trip: data leaves the warehouse and comes back \ + through this loader.\n Lineage and column lineage restart at the source \ + '{source}', and builds and job\n groups run the {kind} before the loader." + ), + help: Some( + "if this is intended, set `allow_warehouse_round_trips = true` under [settings] \ + in sqlbuild_project.toml" + .to_owned(), + ), + }); + } + Ok(()) +} + +/// `kind 'name'` of a Python node, as round-trip errors name it. +fn python_label(context: &SelectionContext<'_>, name: &str) -> String { + let kind: &str = context.nodes.node(name).map_or("node", |node| &node.kind); + format!("{kind} '{name}'") +} + +/// The nearest task or asset upstream of `loader` that reads SQL, with the Python nodes between. +fn sql_reading_upstream<'graph>( + context: &SelectionContext<'graph>, + loader: &str, +) -> Option> { + let mut parents: HashMap<&'graph str, Option<&'graph str>> = HashMap::new(); + let mut queue: VecDeque<&'graph str> = VecDeque::new(); + for upstream in context.nodes.edges(loader, EdgeDirection::Upstream) { + if !parents.contains_key(upstream.as_str()) { + parents.insert(upstream, None); + queue.push_back(upstream); + } + } + while let Some(name) = queue.pop_front() { + if let Some(node) = context.nodes.node(name) + && (node.kind == TASK_KIND || node.kind == ASSET_KIND) + && !node.sql_deps.is_empty() { - let Some(upstream) = context.nodes.node(&upstream_name) else { - continue; - }; - if (upstream.kind == TASK_KIND || upstream.kind == ASSET_KIND) - && !upstream.sql_deps.is_empty() - { - return Err(failure(format!( - "Loader '{}' depends on Python node '{}' which depends on SQL; \ - Python-to-SQL writes must flow through a pre-SQL task/asset -> loader -> \ - source path", - loader.name, upstream.name - ))); + let mut path: Vec<&'graph str> = vec![name]; + let mut current: Option<&'graph str> = parents[name]; + while let Some(parent) = current { + path.push(parent); + current = parents[parent]; + } + path.reverse(); + return Some(path); + } + for upstream in context.nodes.edges(name, EdgeDirection::Upstream) { + if upstream != loader && !parents.contains_key(upstream.as_str()) { + parents.insert(upstream, Some(name)); + queue.push_back(upstream); } } } - Ok(()) + None +} + +/// A node of the combined dependency graph a round trip can close a cycle in. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +enum RoundTripNode { + Python(String), + Sql(ObjectKey), +} + +/// The first cycle through a loader, its Python upstreams, the SQL they read and the source it fills. +fn round_trip_cycle(context: &SelectionContext<'_>, loaders: &[&str]) -> Option> { + let source_loaders: HashMap<&str, &str> = source_loaders(context.sql).into_iter().collect(); + let roots = loaders + .iter() + .map(|name| RoundTripNode::Python((*name).to_owned())); + let path: Vec = find_dependency_cycle(roots, |node| match node { + RoundTripNode::Python(name) => { + let mut next: Vec = context + .nodes + .edges(name, EdgeDirection::Upstream) + .iter() + .map(|upstream| RoundTripNode::Python(upstream.clone())) + .collect(); + if let Some(node) = context.nodes.node(name) + && node.kind != CHECK_KIND + { + next.extend(node.sql_deps.iter().cloned().map(RoundTripNode::Sql)); + } + next + } + RoundTripNode::Sql(key) => { + let mut next: Vec = context + .sql + .graph + .edges(key, EdgeDirection::Upstream) + .iter() + .cloned() + .map(RoundTripNode::Sql) + .collect(); + if key.0 == SOURCE_RESOURCE + && let Some(loader) = source_loaders.get(key.1.as_str()) + { + next.push(RoundTripNode::Python((*loader).to_owned())); + } + next + } + })?; + let last: &RoundTripNode = path.last()?; + let start: usize = path.iter().position(|node| node == last)?; + Some( + path[start..] + .iter() + .map(|node| match node { + RoundTripNode::Python(name) => python_label(context, name), + RoundTripNode::Sql((kind, name)) => format!("{kind} '{name}'"), + }) + .collect(), + ) } /// Selected managed sources have their loaders selected; a Python node may run without its Python upstream, whose latest persisted result it reads. diff --git a/crates/sqlbuild-analysis/src/python_selection/constants.rs b/crates/sqlbuild-analysis/src/python_selection/constants.rs index a563ace884..c199971bf1 100644 --- a/crates/sqlbuild-analysis/src/python_selection/constants.rs +++ b/crates/sqlbuild-analysis/src/python_selection/constants.rs @@ -9,6 +9,7 @@ pub(crate) const TAG_KIND: &str = "tag"; pub(crate) const PATH_KIND: &str = "path"; pub(crate) const TEST_KIND: &str = "test"; pub(crate) const JOB_KIND: &str = "job"; +pub(crate) const NODE_KIND_KIND: &str = "kind"; pub(crate) const SEED_KIND: &str = "seed"; pub(crate) const SOURCE_KIND: &str = "source"; pub(crate) const MODEL_RESOURCE: &str = "model"; diff --git a/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs index 21edb1f7eb..67834ec5dd 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs @@ -23,6 +23,7 @@ pub fn attach_python_jobs( SqlSelectionGraph { graph, sources: sql.sources, + allow_warehouse_round_trips: sql.allow_warehouse_round_trips, }, python, ); diff --git a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs index 4630bc0f8c..39f3584b80 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs @@ -2,6 +2,7 @@ pub mod attach_python_jobs; pub mod published_python_graph; pub mod python_node_graph_indexes; pub mod python_run_key; +pub mod python_sql_graph_error; pub mod resolve_python_node_excludes; pub mod resolve_python_node_selectors; pub mod resolve_python_sql_nodes; diff --git a/crates/sqlbuild-analysis/src/python_selection/main/python_sql_graph_error.rs b/crates/sqlbuild-analysis/src/python_selection/main/python_sql_graph_error.rs new file mode 100644 index 0000000000..561e6ef5f7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/python_sql_graph_error.rs @@ -0,0 +1,14 @@ +//! The first whole-graph boundary error across SQL resources and Python nodes, which compile and selection share. + +use crate::graph::errors::SelectorError; +use crate::python_selection::_helpers::validation::validate_boundaries; +use crate::python_selection::models::{PythonSelectionGraph, SelectionContext, SqlSelectionGraph}; + +/// The error every selection of this graph refuses with, whatever it selects, such as a dependency cycle. +#[must_use] +pub fn python_sql_graph_error( + sql: SqlSelectionGraph<'_>, + python: &PythonSelectionGraph, +) -> Option { + validate_boundaries(&SelectionContext::new(sql, python)).err() +} diff --git a/crates/sqlbuild-analysis/src/python_selection/models.rs b/crates/sqlbuild-analysis/src/python_selection/models.rs index 63b7aacf07..f4c43a3956 100644 --- a/crates/sqlbuild-analysis/src/python_selection/models.rs +++ b/crates/sqlbuild-analysis/src/python_selection/models.rs @@ -32,6 +32,8 @@ pub struct SqlSelectionGraph<'graph> { pub graph: &'graph ProjectGraph, /// `(source name, loader)` for every compiled source. pub sources: &'graph [(String, Option)], + /// `[settings] allow_warehouse_round_trips`: a loader may depend on Python that reads SQL. + pub allow_warehouse_round_trips: bool, } /// The SQL keys and runnable Python node names a run selects, both sorted. diff --git a/crates/sqlbuild-analysis/src/python_selection/tests.rs b/crates/sqlbuild-analysis/src/python_selection/tests.rs index 95e26d2c9f..8e2112d927 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests.rs @@ -2,5 +2,7 @@ mod helpers; #[path = "tests/test_python_selection.rs"] mod python_selection; +#[path = "tests/test_round_trips.rs"] +mod round_trips; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs index 2e8c3134f8..14a0dace69 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs @@ -4,6 +4,7 @@ use crate::graph::main::build_project_graph::build_project_graph; use crate::graph::models::{GraphResource, ProjectGraph}; use crate::python_selection::main::published_python_graph::published_python_graph; use crate::python_selection::main::python_node_graph_indexes::python_node_graph_indexes; +use crate::python_selection::main::python_sql_graph_error::python_sql_graph_error; use crate::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use crate::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; use crate::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection; @@ -11,7 +12,7 @@ use crate::python_selection::models::{ InventoryIndexes, InventoryNode, PublishedPythonNode, PythonSelectionGraph, PythonSelectionNode, RunSelection, SqlSelectionGraph, }; -use crate::python_selection::tests::test_types::{ExpectedRun, SelectorFailure}; +use crate::python_selection::tests::test_types::{ExpectedRun, RoundTripNodeSpec, SelectorFailure}; type OwnedFailure = (String, String); type OwnedRun = (Vec, Vec); @@ -199,6 +200,7 @@ pub(super) fn run_over( SqlSelectionGraph { graph: &graph, sources: &sources, + allow_warehouse_round_trips: false, }, python, &owned(select), @@ -250,3 +252,124 @@ pub(super) fn indexed_folder(relative_path: &str) -> String { pub(super) fn excluded_names(exclude: &[&str]) -> Result, OwnedFailure> { resolve_python_node_excludes(&orders_python_graph(), &owned(exclude)).map_err(failure) } + +/// The orders round-trip SQL graph: `scored_orders` reads `orders` and the `partner_scores` source. +fn round_trip_sql_graph() -> ProjectGraph { + build_project_graph(&[ + GraphResource { + key: key(&("model", "orders")), + deps: Vec::new(), + tags: Vec::new(), + folder: Some(String::new()), + }, + GraphResource { + key: key(&("source", "partner_scores")), + deps: Vec::new(), + tags: Vec::new(), + folder: None, + }, + GraphResource { + key: key(&("model", "scored_orders")), + deps: vec![ + key(&("model", "orders")), + key(&("source", "partner_scores")), + ], + tags: Vec::new(), + folder: Some(String::new()), + }, + ]) +} + +/// A Python graph of `(name, kind, upstream, SQL reads)` nodes. +fn round_trip_python_graph(nodes: &[RoundTripNodeSpec]) -> PythonSelectionGraph { + let inventory: Vec = nodes + .iter() + .map(|(name, _, upstream, _)| InventoryNode { + name: (*name).to_owned(), + tags: Vec::new(), + relative_path: "python/nodes.py".to_owned(), + upstream: owned(upstream), + }) + .collect(); + let indexes: InventoryIndexes = python_node_graph_indexes(&inventory); + PythonSelectionGraph { + nodes: nodes + .iter() + .map(|(name, kind, _, sql_deps)| PythonSelectionNode { + name: (*name).to_owned(), + kind: (*kind).to_owned(), + sql_deps: sql_deps.iter().map(key).collect(), + }) + .collect(), + upstream: indexes.upstream, + downstream: indexes.downstream, + tags: indexes.tags, + paths: indexes.paths, + } +} + +/// The run selection of a round-trip project, or the selector error's `(code, message, help)`. +pub(super) fn round_trip_run( + nodes: &[RoundTripNodeSpec], + allow_warehouse_round_trips: bool, + select: &[&str], +) -> Result { + let graph: ProjectGraph = round_trip_sql_graph(); + let sources: Vec<(String, Option)> = vec![( + "partner_scores".to_owned(), + Some("partner_scores".to_owned()), + )]; + resolve_python_sql_run_selection( + SqlSelectionGraph { + graph: &graph, + sources: &sources, + allow_warehouse_round_trips, + }, + &round_trip_python_graph(nodes), + &owned(select), + &[], + ) + .map( + |RunSelection { + sql_keys, + python_node_names, + }| (sql_keys, python_node_names), + ) + .map_err(|error| { + ( + error.code.to_owned(), + error.message, + error.help.unwrap_or_default(), + ) + }) +} + +/// A round-trip case's expected selection, or its expected `(code, message, help)`, owned. +pub(super) fn expected_round_trip( + expected: Result, +) -> Result { + expected + .map(|(keys, names)| (keys.iter().map(key).collect(), owned(names))) + .map_err(|(code, message, help)| (code.to_owned(), message.to_owned(), help.to_owned())) +} + +/// The whole-graph error of a round-trip project, when it has one; the loader fills its source only when `filled`. +pub(super) fn round_trip_graph_error( + nodes: &[RoundTripNodeSpec], + (filled, allow_warehouse_round_trips): (bool, bool), +) -> Option { + let graph: ProjectGraph = round_trip_sql_graph(); + let sources: Vec<(String, Option)> = vec![( + "partner_scores".to_owned(), + filled.then(|| "partner_scores".to_owned()), + )]; + python_sql_graph_error( + SqlSelectionGraph { + graph: &graph, + sources: &sources, + allow_warehouse_round_trips, + }, + &round_trip_python_graph(nodes), + ) + .map(|error| error.message) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs index fad17b2ca9..2a2a7e0231 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs @@ -183,6 +183,48 @@ fn given_run_selectors_when_resolving_then_selection_matches_python() { select it as source:raw_orders", )), }, + RunSelectionTestCase { + description: "kind selects every SQL resource of one kind", + select: &["kind:model"], + exclude: &[], + expected_selection: Ok((&[("model", "orders")], &[])), + }, + RunSelectionTestCase { + description: "kind selects every Python node of one kind", + select: &["kind:task"], + exclude: &[], + expected_selection: Ok((&[], &["prepare_orders"])), + }, + RunSelectionTestCase { + description: "+ expands only the comma part it marks: upstream of a node, then only its sources", + select: &["+export_orders,kind:source"], + exclude: &[], + expected_selection: Ok((&[("source", "raw_orders")], &[])), + }, + RunSelectionTestCase { + description: "+ on a kind part expands that part before the intersection", + select: &["kind:source,+orders"], + exclude: &[], + expected_selection: Ok((&[("source", "raw_orders")], &[])), + }, + RunSelectionTestCase { + description: "kind excludes like tag", + select: &["tag:daily"], + exclude: &["kind:asset"], + expected_selection: Ok((&[("model", "orders")], &["prepare_orders"])), + }, + RunSelectionTestCase { + description: "an unknown kind is refused", + select: &["kind:modle"], + exclude: &[], + expected_selection: Err(("S007", "unknown node kind 'modle'")), + }, + RunSelectionTestCase { + description: "a kind the project has no nodes of is refused like an unknown tag", + select: &["kind:udf"], + exclude: &[], + expected_selection: Err(("S000", "no nodes found of kind 'udf'")), + }, RunSelectionTestCase { description: "tag on neither side", select: &["tag:missing"], diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_round_trips.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_round_trips.rs new file mode 100644 index 0000000000..be5ce9aa49 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_round_trips.rs @@ -0,0 +1,267 @@ +use crate::python_selection::tests::helpers::{ + expected_round_trip, round_trip_graph_error, round_trip_run, +}; +use crate::python_selection::tests::test_types::{ + CycleTestCase, RoundTripNodeSpec, RoundTripTestCase, +}; + +const DIRECT_CYCLE: &str = "loader 'partner_scores' is in a dependency cycle: loader 'partner_scores' -> \ + task 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> \ + loader 'partner_scores'"; +const SCORE_ORDERS: RoundTripNodeSpec = ("score_orders", "task", &[], &[("model", "orders")]); +const PARTNER_SCORES: RoundTripNodeSpec = ("partner_scores", "loader", &["score_orders"], &[]); +const HELP: &str = "if this is intended, set `allow_warehouse_round_trips = true` under [settings] \ + in sqlbuild_project.toml"; +const CYCLE_HELP: &str = "each arrow points to what a node depends on; a task or asset upstream \ + of a loader cannot read anything built from the source that loader fills"; + +#[test] +fn given_a_loader_depending_on_python_that_reads_sql_when_selecting_then_the_setting_decides() { + let test_cases = [ + RoundTripTestCase { + description: "a direct SQL-reading task is refused by default", + nodes: &[SCORE_ORDERS, PARTNER_SCORES], + allow_warehouse_round_trips: false, + select: &[], + expected_selection: Err(( + "S000", + "loader 'partner_scores' depends on task 'score_orders', which reads SQL\n \ + This is a warehouse round trip: data leaves the warehouse and comes back through \ + this loader.\n Lineage and column lineage restart at the source \ + 'partner_scores', and builds and job\n groups run the task before the loader.", + HELP, + )), + }, + RoundTripTestCase { + description: "a transitive SQL-reading asset is refused by default and named with its path", + nodes: &[ + ("score_orders", "asset", &[], &[("model", "orders")]), + ("package_scores", "task", &["score_orders"], &[]), + ("partner_scores", "loader", &["package_scores"], &[]), + ], + allow_warehouse_round_trips: false, + select: &[], + expected_selection: Err(( + "S000", + "loader 'partner_scores' depends on asset 'score_orders' (through task \ + 'package_scores'), which reads SQL\n This is a warehouse round trip: data \ + leaves the warehouse and comes back through this loader.\n Lineage and column \ + lineage restart at the source 'partner_scores', and builds and job\n groups run \ + the asset before the loader.", + HELP, + )), + }, + RoundTripTestCase { + description: "the setting allows the round trip, which upstream selection follows", + nodes: &[SCORE_ORDERS, PARTNER_SCORES], + allow_warehouse_round_trips: true, + select: &["+scored_orders"], + expected_selection: Ok(( + &[ + ("model", "orders"), + ("model", "scored_orders"), + ("source", "partner_scores"), + ], + &["partner_scores", "score_orders"], + )), + }, + RoundTripTestCase { + description: "a loader depending on a task that reads no SQL is always allowed", + nodes: &[ + ("fetch_scores", "task", &[], &[]), + ("partner_scores", "loader", &["fetch_scores"], &[]), + ], + allow_warehouse_round_trips: false, + select: &["+scored_orders"], + expected_selection: Ok(( + &[ + ("model", "orders"), + ("model", "scored_orders"), + ("source", "partner_scores"), + ], + &["fetch_scores", "partner_scores"], + )), + }, + RoundTripTestCase { + description: "a task reading what its downstream loader's source builds is a cycle", + nodes: &[ + ("score_orders", "task", &[], &[("model", "scored_orders")]), + PARTNER_SCORES, + ], + allow_warehouse_round_trips: true, + select: &[], + expected_selection: Err(( + "S000", + "loader 'partner_scores' is in a dependency cycle: loader 'partner_scores' -> \ + task 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> \ + loader 'partner_scores'", + CYCLE_HELP, + )), + }, + RoundTripTestCase { + description: "a cycle is reported before the setting is consulted", + nodes: &[ + ("score_orders", "task", &[], &[("model", "scored_orders")]), + PARTNER_SCORES, + ], + allow_warehouse_round_trips: false, + select: &[], + expected_selection: Err(( + "S000", + "loader 'partner_scores' is in a dependency cycle: loader 'partner_scores' -> \ + task 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> \ + loader 'partner_scores'", + CYCLE_HELP, + )), + }, + ]; + + for test_case in test_cases { + assert_eq!( + round_trip_run( + test_case.nodes, + test_case.allow_warehouse_round_trips, + test_case.select + ), + expected_round_trip(test_case.expected_selection), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_whole_graph_boundaries_when_checked_without_a_selection_then_each_refusal_is_reported() { + let test_cases = [ + CycleTestCase { + description: "a loader's depends_on closes the cycle", + nodes: &[ + ("score_orders", "task", &[], &[("model", "scored_orders")]), + ("partner_scores", "loader", &["score_orders"], &[]), + ], + filled: true, + allow_warehouse_round_trips: true, + expected_error: Some(DIRECT_CYCLE), + }, + CycleTestCase { + description: "without the loader's depends_on there is no cycle", + nodes: &[ + ("score_orders", "task", &[], &[("model", "scored_orders")]), + ("partner_scores", "loader", &[], &[]), + ], + filled: true, + allow_warehouse_round_trips: true, + expected_error: None, + }, + CycleTestCase { + description: "a task's depends_on on another task closes the cycle", + nodes: &[ + ("score_orders", "task", &[], &[("model", "scored_orders")]), + ("package_scores", "task", &["score_orders"], &[]), + ("partner_scores", "loader", &["package_scores"], &[]), + ], + filled: true, + allow_warehouse_round_trips: true, + expected_error: Some( + "loader 'partner_scores' is in a dependency cycle: loader 'partner_scores' -> \ + task 'package_scores' -> task 'score_orders' -> model 'scored_orders' -> \ + source 'partner_scores' -> loader 'partner_scores'", + ), + }, + CycleTestCase { + description: "an asset's depends_on closes the cycle", + nodes: &[ + ("score_orders", "task", &[], &[("model", "scored_orders")]), + ("scores_file", "asset", &["score_orders"], &[]), + ("partner_scores", "loader", &["scores_file"], &[]), + ], + filled: true, + allow_warehouse_round_trips: true, + expected_error: Some( + "loader 'partner_scores' is in a dependency cycle: loader 'partner_scores' -> \ + asset 'scores_file' -> task 'score_orders' -> model 'scored_orders' -> \ + source 'partner_scores' -> loader 'partner_scores'", + ), + }, + CycleTestCase { + description: "without the task's depends_on there is no cycle", + nodes: &[ + ("score_orders", "task", &[], &[("model", "scored_orders")]), + ("package_scores", "task", &[], &[]), + ("partner_scores", "loader", &["package_scores"], &[]), + ], + filled: true, + allow_warehouse_round_trips: true, + expected_error: None, + }, + CycleTestCase { + description: "a Python node's dependency model, through a model ref to the source, closes the cycle", + nodes: &[ + ("score_orders", "task", &[], &[("model", "scored_orders")]), + ("partner_scores", "loader", &["score_orders"], &[]), + ], + filled: true, + allow_warehouse_round_trips: true, + expected_error: Some(DIRECT_CYCLE), + }, + CycleTestCase { + description: "a dependency model whose refs never reach the source closes no cycle", + nodes: &[ + ("score_orders", "task", &[], &[("model", "orders")]), + ("partner_scores", "loader", &["score_orders"], &[]), + ], + filled: true, + allow_warehouse_round_trips: true, + expected_error: None, + }, + CycleTestCase { + description: "a loader downstream of a task that reads SQL, without the setting, is a round trip", + nodes: &[SCORE_ORDERS, PARTNER_SCORES], + filled: true, + allow_warehouse_round_trips: false, + expected_error: Some( + "loader 'partner_scores' depends on task 'score_orders', which reads SQL\n \ + This is a warehouse round trip: data leaves the warehouse and comes back through \ + this loader.\n Lineage and column lineage restart at the source \ + 'partner_scores', and builds and job\n groups run the task before the loader.", + ), + }, + CycleTestCase { + description: "a task depending on the terminal loader instead of its source", + nodes: &[ + ("fetch_scores", "task", &[], &[]), + ("partner_scores", "loader", &["fetch_scores"], &[]), + ("report_scores", "task", &["partner_scores"], &[]), + ], + filled: true, + allow_warehouse_round_trips: false, + expected_error: Some( + "Python node 'report_scores' depends on terminal loader 'partner_scores'; \ + depend on source 'partner_scores' instead", + ), + }, + CycleTestCase { + description: "without the source's loader there is no cycle", + nodes: &[ + ("score_orders", "task", &[], &[("model", "scored_orders")]), + ("partner_scores", "loader", &["score_orders"], &[]), + ], + filled: false, + allow_warehouse_round_trips: true, + expected_error: None, + }, + ]; + for test_case in test_cases { + let error = round_trip_graph_error( + test_case.nodes, + (test_case.filled, test_case.allow_warehouse_round_trips), + ); + + assert_eq!( + error.as_deref(), + test_case.expected_error, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_types.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_types.rs index 67674006a1..4083e0eba1 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_types.rs @@ -26,3 +26,30 @@ pub(super) type ExpectedRun = ( /// `(code, message)` of the `PlannerInputError` Python raises. pub(super) type SelectorFailure = (&'static str, &'static str); + +pub(super) struct RoundTripTestCase { + pub(super) description: &'static str, + /// `(name, kind, upstream Python nodes, SQL reads)` of each Python node. + pub(super) nodes: &'static [RoundTripNodeSpec], + pub(super) allow_warehouse_round_trips: bool, + pub(super) select: &'static [&'static str], + pub(super) expected_selection: Result, +} + +/// `(name, kind, upstream Python nodes, SQL reads)`. +pub(super) type RoundTripNodeSpec = ( + &'static str, + &'static str, + &'static [&'static str], + &'static [(&'static str, &'static str)], +); + +pub(super) struct CycleTestCase { + pub(super) description: &'static str, + /// `(name, kind, upstream Python nodes, SQL reads)` of each Python node. + pub(super) nodes: &'static [RoundTripNodeSpec], + /// Whether the `partner_scores` source names `partner_scores` as its loader. + pub(super) filled: bool, + pub(super) allow_warehouse_round_trips: bool, + pub(super) expected_error: Option<&'static str>, +} diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs index 6c4454814a..816b959d3f 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs @@ -281,7 +281,7 @@ fn is_sql_test(resource: &PlannerResource) -> bool { resource.key.split_once(':').map(|(kind, _)| kind) == Some(SQL_TEST_KIND) } -/// The published keys the selectors match; Python nodes only when the command runs them. +/// The published keys the selectors match; Python nodes only when the command runs them, round trips allowed because the launched command enforces the project's setting. fn resolved( (resources, jobs): (&[PlannerResource], &PublishedJobs), select: &[String], @@ -322,6 +322,7 @@ fn resolved( SqlSelectionGraph { graph: &graph, sources: &sources, + allow_warehouse_round_trips: true, }, &python_graph, select, diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index fe516eaeb3..79cf935f9a 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -26,7 +26,7 @@ use crate::project::_helpers::values::is_truthy; use crate::project::constants::{ ALLOW_MISSING_SECRETS_KEY, COLUMN_CONTRACT_MODES, LEGACY_COLUMN_CONTRACT_MODE_KEY, LEGACY_EVENT_EXPORTERS_KEY, LOCAL_CONFIG_FILENAME, QUERY_CHANGE_TRACKING_KEY, - REQUIRE_SQL_ANALYSIS_KEY, SQL_ANALYSIS_KEY, + REQUIRE_SQL_ANALYSIS_KEY, SHARED_POLICY_SETTINGS, SQL_ANALYSIS_KEY, }; use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; use crate::project::models::{ @@ -316,15 +316,15 @@ pub(crate) fn project_config(paths: &ConfigPaths) -> Result Ok(object("ProjectConfig", project_fields(payload)?)) } -fn shared_policy_failure(value: &ConfigValue) -> ConfigFailure { +fn shared_policy_failure(key: &str, value: &ConfigValue) -> ConfigFailure { let help = join_helps(&[ - format!("remove {REQUIRE_SQL_ANALYSIS_KEY} from {LOCAL_CONFIG_FILENAME}"), + format!("remove {key} from {LOCAL_CONFIG_FILENAME}"), setting_help( "to change the policy for everyone", &Setting { file_name: PROJECT_CONFIG_FILENAME, section: SETTINGS_SECTION, - key: REQUIRE_SQL_ANALYSIS_KEY, + key, value: SettingValue::Bool(!is_truthy(value)), }, ), @@ -332,8 +332,8 @@ fn shared_policy_failure(value: &ConfigValue) -> ConfigFailure { ConfigFailure::at( LOCAL_FILE, format!( - "cannot set [settings] {REQUIRE_SQL_ANALYSIS_KEY}; it is a shared project policy \ - that only {PROJECT_CONFIG_FILENAME} can set" + "cannot set [settings] {key}; it is a shared project policy that only \ + {PROJECT_CONFIG_FILENAME} can set" ), ) .with_help(help) @@ -347,9 +347,12 @@ fn local_settings(payload: Option<&ConfigValue>) -> Loaded { label: "settings", }, )?; - if has(mapping, REQUIRE_SQL_ANALYSIS_KEY) { - let value = get(mapping, REQUIRE_SQL_ANALYSIS_KEY).unwrap_or(&ConfigValue::Null); - return Err(shared_policy_failure(value)); + if let Some(key) = SHARED_POLICY_SETTINGS + .into_iter() + .find(|key| has(mapping, key)) + { + let value = get(mapping, key).unwrap_or(&ConfigValue::Null); + return Err(shared_policy_failure(key, value)); } let overrides = setting_overrides(mapping); Ok(vec![ diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 5d966d1923..741c34418b 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -11,9 +11,9 @@ use crate::project::_helpers::keys::{Place, reject_removed_virtual_keys, validat use crate::project::_helpers::targets::seconds_value; use crate::project::_helpers::values::{int_below, python_equal}; use crate::project::constants::{ - ALLOW_MISSING_SECRETS_KEY, DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, - DELTA_COMPILE_KEY, DISABLED_VALUE, SCENARIO_TIMEOUT_KEY, TEST_TIMEOUT_KEY, - UNIT_TEST_MAX_REAL_MODELS_KEY, + ALLOW_MISSING_SECRETS_KEY, ALLOW_WAREHOUSE_ROUND_TRIPS_KEY, DEFAULT_TEST_TIMEOUT, + DEFAULT_UNIT_TEST_MAX_REAL_MODELS, DELTA_COMPILE_KEY, DISABLED_VALUE, SCENARIO_TIMEOUT_KEY, + TEST_TIMEOUT_KEY, UNIT_TEST_MAX_REAL_MODELS_KEY, }; use crate::project::constants::{ AUDIT_SEVERITIES, CHECK_DECLARED_COLUMNS_KEY, COLLECTION_RENDERINGS, COLUMN_CONTRACT_MODES, @@ -285,6 +285,14 @@ pub(crate) fn load_settings(payload: Option<&ConfigValue>, file: ConfigFile) -> false, )?)), ), + ( + ALLOW_WAREHOUSE_ROUND_TRIPS_KEY, + Built::Value(ConfigValue::Bool(optional_bool( + mapping, + ALLOW_WAREHOUSE_ROUND_TRIPS_KEY, + false, + )?)), + ), ]; let mut fields = fields; for (key, seconds_key, default) in [ diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 722e754a54..d19ddf66f7 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -9,6 +9,10 @@ pub(crate) const CHECK_DECLARED_COLUMNS_KEY: &str = "check_declared_columns"; /// The deprecated alias of `check_declared_columns`. pub(crate) const LEGACY_COLUMN_CONTRACT_MODE_KEY: &str = "column_contract_mode"; pub(crate) const REQUIRE_SQL_ANALYSIS_KEY: &str = "require_sql_analysis"; +pub(crate) const ALLOW_WAREHOUSE_ROUND_TRIPS_KEY: &str = "allow_warehouse_round_trips"; +/// The `[settings]` keys that are shared project policy, which only the project file may set. +pub(crate) const SHARED_POLICY_SETTINGS: [&str; 2] = + [REQUIRE_SQL_ANALYSIS_KEY, ALLOW_WAREHOUSE_ROUND_TRIPS_KEY]; pub(crate) const QUERY_CHANGE_TRACKING_KEY: &str = "query_change_tracking"; pub(crate) const CONCURRENCY_KEY: &str = "concurrency"; pub(crate) const LEGACY_CONCURRENCY_KEY: &str = "max_concurrency"; @@ -50,7 +54,7 @@ pub(crate) const DELTA_COMPILE_KEY: &str = "delta_compile"; pub(crate) const ALLOW_MISSING_SECRETS_KEY: &str = "allow_missing_secrets"; pub(crate) const SCENARIO_TIMEOUT_KEY: &str = "scenario_timeout"; pub(crate) const DEFAULT_TEST_TIMEOUT: &str = "30s"; -pub(crate) const SETTINGS_KEYS: [&str; 16] = [ +pub(crate) const SETTINGS_KEYS: [&str; 17] = [ "sql_analysis", "require_sql_analysis", "query_change_tracking", @@ -67,6 +71,7 @@ pub(crate) const SETTINGS_KEYS: [&str; 16] = [ "scenario_timeout", "delta_compile", "allow_missing_secrets", + "allow_warehouse_round_trips", ]; pub(crate) const DEFAULTS_KEYS: [&str; 33] = [ "append_cursor_inclusive", diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 0fb8d81691..a41c8a98f4 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -142,6 +142,21 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { ), ), }, + ProjectFailureTestCase { + description: "a local file cannot allow warehouse round trips", + project: PROJECT, + local: "[settings]\nallow_warehouse_round_trips = true\n", + expected_failure: failure_at( + ConfigFile::Local, + "cannot set [settings] allow_warehouse_round_trips; it is a shared project policy \ + that only sqlbuild_project.toml can set", + Some( + "remove allow_warehouse_round_trips from sqlbuild_local.toml\n = help: to \ + change the policy for everyone, set this in sqlbuild_project.toml:\n \ + [settings]\n\x20 allow_warehouse_round_trips = false", + ), + ), + }, ProjectFailureTestCase { description: "a path default normalizing to an existing key is rejected", project: "name = 'orders'\nadapter = 'duckdb'\n[path_defaults.staging]\n[path_defaults.' staging ']\n", diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs index fa83395d14..9c1b64b6b8 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs @@ -817,6 +817,7 @@ const R_PYTHON_SQL_RUN_LIFECYCLE_PLAN: RecordSpec = RecordSpec { FieldSpec::required("ingress_python_node_names"), FieldSpec::required("ingress_loader_names"), FieldSpec::required("read_side_python_node_names"), + FieldSpec::new("round_trip_loader_names", FieldDefault::EmptyFrozenset), ], }; const R_PYTHON_IDENTITY_DEPENDENCY: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index 33fbe90f14..e9bba6a067 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -232,6 +232,7 @@ const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("scenario_timeout_seconds", FieldDefault::None), FieldSpec::new("delta_compile", FieldDefault::True), FieldSpec::new("allow_missing_secrets", FieldDefault::False), + FieldSpec::new("allow_warehouse_round_trips", FieldDefault::False), ], }; const R_SCOPES_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs index 40c1ae83e3..04d9a882b3 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs @@ -18,6 +18,8 @@ use crate::rules::main::rule::evaluate_rules::evaluate_rules; use crate::rules::models::EarlyRulesScope; use crate::rules::models::EarlyRulesSettings; use crate::rules::models::EarlySqlLint; +use sqlbuild_analysis::graph::constants::PLANNER_DEFAULT_CODE; +use sqlbuild_analysis::graph::errors::SelectorError; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -35,7 +37,9 @@ use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_project::contracts::main::evaluate_model_contracts::evaluate_model_contracts; +use sqlbuild_python_project::graph::main::python_nodes::project_graph_error::project_graph_error; use sqlbuild_python_project::graph::models::CompiledProjectGraph; +use sqlbuild_python_project::planner_identity::main::execution_cycle::execution_cycle; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; use sqlbuild_python_project::project_snapshot::main::job_group_cycles::job_group_cycles; @@ -239,10 +243,16 @@ fn analyzed<'py>( )?)?; let selected_keys: Bound<'py, PyAny> = select_project_graph(&graph, &driver.select, &driver.exclude)?.into_any(); + let boundary: Vec> = if graph_failed { + Vec::new() + } else { + boundary_diagnostics(&graph, &discovered)? + }; let core_diagnostics: Bound<'py, PyTuple> = PyTuple::new( py, native_field(&project, "diagnostics")? .try_iter()? + .chain(boundary.into_iter().map(Ok)) .chain(native_field(&contracts.result, "diagnostics")?.try_iter()?) .collect::>>()?, )?; @@ -560,6 +570,51 @@ impl<'py> Driver<'_, 'py> { } } +/// Compile errors for what every plan refuses: an execution-order cycle and the first SQL/Python boundary error. +fn boundary_diagnostics<'py>( + graph: &Bound<'py, PyAny>, + discovered: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = graph.py(); + let mut refusals: Vec = Vec::new(); + if let Some(message) = execution_cycle(&native_field(graph, "project")?)? { + refusals.push(SelectorError { + code: PLANNER_DEFAULT_CODE, + message, + help: None, + }); + } + refusals.extend(project_graph_error(graph, discovered)?); + let none: Bound<'py, PyAny> = py.None().into_bound(py); + refusals + .into_iter() + .map(|refusal| { + new_record_with( + py, + RecordId::CompilerDiagnostic, + &[ + ( + "phase", + enum_member(py, RecordId::DiagnosticPhase, "COMPILE")?, + ), + ( + "severity", + enum_member(py, RecordId::DiagnosticSeverity, "ERROR")?, + ), + ("code", PyString::new(py, refusal.code).into_any()), + ("message", PyString::new(py, &refusal.message).into_any()), + ("path", none.clone()), + ("line", none.clone()), + ("column", none.clone()), + ("help", refusal.help.into_pyobject(py)?.into_any()), + ("notes", PyTuple::empty(py).into_any()), + ("affected_rules", PyTuple::empty(py).into_any()), + ], + ) + }) + .collect() +} + /// The core diagnostics followed by one rule-phase error per rule finding; lint-unavailable ones only warn. fn with_rule_diagnostics<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs index 9d7502e05a..ca64cee07a 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs @@ -125,11 +125,25 @@ fn inventory_catalog(inventory: &OrchestrationInventory, target: Option<&str>) - }, ); } + let job_targets: BTreeMap<&str, &[String]> = inventory + .jobs + .iter() + .map(|(_, job)| (job.run.name.value.as_str(), job.run.targets.as_slice())) + .collect(); for (_, group) in &inventory.groups { add( &group.run, JobSelection::Group { - members: group.jobs.iter().map(|job| job.value.clone()).collect(), + members: group + .jobs + .iter() + .filter(|job| { + job_targets + .get(job.value.as_str()) + .is_none_or(|targets| declared_for(targets)) + }) + .map(|job| job.value.clone()) + .collect(), }, ); } diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs index 217782ff6b..be9505fd58 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_lifecycle.rs @@ -1,5 +1,6 @@ //! Lifecycle regions of a run's selected Python nodes, classified natively. +use crate::graph::_helpers::python_nodes::python_selection::sql_deps; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::python_plan::models::LifecycleNode; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -7,5 +8,6 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; pub(crate) fn lifecycle_node(node: &Bound<'_, PyAny>) -> PyResult { Ok(LifecycleNode { kind: native_field(node, "kind")?.extract()?, + sql_deps: sql_deps(node)?, }) } diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs index 1cd6175c21..6a23dbe2a3 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs @@ -74,7 +74,8 @@ fn name_rows(mapping: &Bound<'_, PyAny>) -> PyResult { Ok(rows) } -fn selection_node(node: &Bound<'_, PyAny>) -> PyResult { +/// The `(kind, name)` keys of the SQL resources a Python node reads, in authored order. +pub(crate) fn sql_deps(node: &Bound<'_, PyAny>) -> PyResult> { let mut sql_deps: Vec = Vec::new(); for reference in native_field(node, "sql_deps")?.try_iter()? { let reference: Bound<'_, PyAny> = reference?; @@ -85,10 +86,14 @@ fn selection_node(node: &Bound<'_, PyAny>) -> PyResult { native_field(&reference, "name")?.extract()?, )); } + Ok(sql_deps) +} + +fn selection_node(node: &Bound<'_, PyAny>) -> PyResult { Ok(PythonSelectionNode { name: native_field(node, "name")?.extract()?, kind: native_field(node, "kind")?.getattr("value")?.extract()?, - sql_deps, + sql_deps: sql_deps(node)?, }) } @@ -128,6 +133,15 @@ pub(crate) fn project_sources( .collect() } +/// `[settings] allow_warehouse_round_trips` of a compiled project. +pub(crate) fn allows_round_trips(project: &Bound<'_, PyAny>) -> PyResult { + native_field( + &native_field(project, "settings")?, + "allow_warehouse_round_trips", + )? + .extract() +} + /// The Python node names `select` minus `exclude` resolve to; every node when unselected. #[pyfunction] #[pyo3(signature = (*, graph, select, exclude))] @@ -152,6 +166,7 @@ pub(crate) fn resolve_python_sql_run_selection( let project_graph: &Bound<'_, CompiledProjectGraph> = project_graph.cast()?; let native: &Bound<'_, NativeProjectGraph> = project_graph.get().native(py); let sources: Vec<(String, Option)> = project_sources(project_graph.get().project(py))?; + let allow_warehouse_round_trips: bool = allows_round_trips(project_graph.get().project(py))?; let nodes: PythonSelectionGraph = selection_graph(python_graph)?; let node_names: Vec<&str> = nodes.nodes.iter().map(|node| node.name.as_str()).collect(); let selectors: Vec = select.iter().chain(&exclude).cloned().collect(); @@ -161,6 +176,7 @@ pub(crate) fn resolve_python_sql_run_selection( SqlSelectionGraph { graph: native.get().graph(), sources: &sources, + allow_warehouse_round_trips, }, &nodes, catalog, @@ -172,6 +188,7 @@ pub(crate) fn resolve_python_sql_run_selection( SqlSelectionGraph { graph: &graph, sources: &sources, + allow_warehouse_round_trips, }, &nodes, &select, diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs index 3c85e9cb2d..7fd2634311 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs @@ -1,15 +1,12 @@ //! `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. -use crate::graph::_helpers::python_nodes::python_lifecycle::lifecycle_node; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use crate::graph::main::python_nodes::native_run_lifecycle::native_run_lifecycle; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::{PyDict, PyFrozenSet}; -use sqlbuild_analysis::python_plan::main::run_lifecycle::run_lifecycle; -use sqlbuild_analysis::python_plan::models::{LifecycleNode, RunLifecycle}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; +use sqlbuild_analysis::python_plan::models::RunLifecycle; use sqlbuild_python_base::records::main::construct::new_record::new_record; -use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; -use std::collections::{BTreeSet, HashMap}; /// `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. #[pyfunction] @@ -19,31 +16,7 @@ pub fn build_python_sql_run_lifecycle<'py>( python_graph: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = selection.py(); - let selected: BTreeSet = native_field(selection, "python_node_names")? - .try_iter()? - .map(|name| name?.extract()) - .collect::>()?; - let mut nodes: HashMap = HashMap::new(); - for item in native_field(python_graph, "nodes_by_name")? - .call_method0("items")? - .try_iter()? - { - let (name, node): (String, Bound<'py, PyAny>) = item?.extract()?; - nodes.insert(name, lifecycle_node(&node)?); - } - let mut upstream: HashMap> = HashMap::new(); - for item in native_field(python_graph, "upstream_deps")? - .call_method0("items")? - .try_iter()? - { - let (name, deps): (String, Bound<'py, PyAny>) = item?.extract()?; - let deps: Vec = deps - .try_iter()? - .map(|dep| dep?.extract()) - .collect::>()?; - upstream.insert(name, deps); - } - let lifecycle: RunLifecycle = py.detach(|| run_lifecycle(&selected, &nodes, &upstream)); + let lifecycle: RunLifecycle = native_run_lifecycle(selection, python_graph)?; let arguments: Bound<'py, PyDict> = PyDict::new(py); arguments.set_item( "ingress_python_node_names", @@ -57,6 +30,16 @@ pub fn build_python_sql_run_lifecycle<'py>( "read_side_python_node_names", PyFrozenSet::new(py, lifecycle.read_side_names)?, )?; + arguments.set_item( + "round_trip_loader_names", + PyFrozenSet::new( + py, + lifecycle + .round_trip_loaders + .into_iter() + .map(|loader| loader.name), + )?, + )?; new_record( py, RecordId::PythonSqlRunLifecyclePlan, diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs index 07eb988543..e0c9f75bcb 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/mod.rs @@ -1,4 +1,6 @@ pub mod build_python_sql_run_lifecycle; +pub mod native_run_lifecycle; +pub mod project_graph_error; pub mod python_node_graph; pub mod python_node_names; pub mod register_python_lifecycle; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/native_run_lifecycle.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/native_run_lifecycle.rs new file mode 100644 index 0000000000..7635c2a378 --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/native_run_lifecycle.rs @@ -0,0 +1,41 @@ +//! `native_run_lifecycle(selection, python_graph)`: the lifecycle regions of a run's Python nodes. + +use crate::graph::_helpers::python_nodes::python_lifecycle::lifecycle_node; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_analysis::python_plan::main::run_lifecycle::run_lifecycle; +use sqlbuild_analysis::python_plan::models::{LifecycleNode, RunLifecycle}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::{BTreeSet, HashMap}; + +/// The lifecycle regions of a `PythonSqlRunSelection` over a `PythonNodeGraph`. +pub fn native_run_lifecycle( + selection: &Bound<'_, PyAny>, + python_graph: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = selection.py(); + let selected: BTreeSet = native_field(selection, "python_node_names")? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?; + let mut nodes: HashMap = HashMap::new(); + for item in native_field(python_graph, "nodes_by_name")? + .call_method0("items")? + .try_iter()? + { + let (name, node): (String, Bound<'_, PyAny>) = item?.extract()?; + nodes.insert(name, lifecycle_node(&node)?); + } + let mut upstream: HashMap> = HashMap::new(); + for item in native_field(python_graph, "upstream_deps")? + .call_method0("items")? + .try_iter()? + { + let (name, deps): (String, Bound<'_, PyAny>) = item?.extract()?; + let deps: Vec = deps + .try_iter()? + .map(|dep| dep?.extract()) + .collect::>()?; + upstream.insert(name, deps); + } + Ok(py.detach(|| run_lifecycle(&selected, &nodes, &upstream))) +} diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/project_graph_error.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/project_graph_error.rs new file mode 100644 index 0000000000..6fc4c5c0ea --- /dev/null +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/project_graph_error.rs @@ -0,0 +1,35 @@ +//! `project_graph_error(project_graph, discovered_inputs)`: the first whole-graph boundary error. + +use crate::graph::_helpers::python_nodes::python_selection::{ + allows_round_trips, project_sources, selection_graph, +}; +use crate::graph::main::python_nodes::python_node_graph::python_node_graph; +use crate::graph::models::{CompiledProjectGraph, NativeProjectGraph}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_analysis::graph::errors::SelectorError; +use sqlbuild_analysis::python_selection::main::python_sql_graph_error::python_sql_graph_error; +use sqlbuild_analysis::python_selection::models::{PythonSelectionGraph, SqlSelectionGraph}; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; + +/// The project's first whole-graph boundary error across SQL resources and Python nodes, as every selection reports it. +pub fn project_graph_error( + project_graph: &Bound<'_, PyAny>, + discovered_inputs: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = project_graph.py(); + let project_graph: &Bound<'_, CompiledProjectGraph> = project_graph.cast()?; + let project: &Bound<'_, PyAny> = project_graph.get().project(py); + let sources: Vec<(String, Option)> = project_sources(project)?; + let nodes: PythonSelectionGraph = selection_graph(&python_node_graph(discovered_inputs)?)?; + let native: &Bound<'_, NativeProjectGraph> = project_graph.get().native(py); + compiler_guard(|| { + Ok(python_sql_graph_error( + SqlSelectionGraph { + graph: native.get().graph(), + sources: &sources, + allow_warehouse_round_trips: allows_round_trips(project)?, + }, + &nodes, + )) + }) +} diff --git a/crates/sqlbuild-python-project/src/graph/models.rs b/crates/sqlbuild-python-project/src/graph/models.rs index d47fccc589..52e20da977 100644 --- a/crates/sqlbuild-python-project/src/graph/models.rs +++ b/crates/sqlbuild-python-project/src/graph/models.rs @@ -6,6 +6,7 @@ use crate::graph::_helpers::project::project_graph::UniqueIdMap; use crate::graph::_helpers::project::project_graph::dependency_edges; use crate::graph::_helpers::project::project_graph::outcome; use crate::graph::_helpers::project::views::lineage_graph_views; +use crate::graph::_helpers::python_nodes::python_selection::allows_round_trips; use crate::graph::_helpers::python_nodes::python_selection::project_sources; use crate::graph::_helpers::python_nodes::python_selection::selection_graph; use crate::graph::constants::LOADER_KIND; @@ -395,6 +396,7 @@ impl NativeProjectGraph { #[derive(Debug)] pub struct ProjectSelection { pub(crate) sources: Vec<(String, Option)>, + allow_warehouse_round_trips: bool, pub nodes: PythonSelectionGraph, } @@ -402,6 +404,7 @@ impl ProjectSelection { pub fn new(project: &Bound<'_, PyAny>, discovered_inputs: &Bound<'_, PyAny>) -> PyResult { Ok(Self { sources: project_sources(project)?, + allow_warehouse_round_trips: allows_round_trips(project)?, nodes: selection_graph(&python_node_graph(discovered_inputs)?)?, }) } @@ -498,6 +501,7 @@ impl ProjectSelection { SqlSelectionGraph { graph, sources: &self.sources, + allow_warehouse_round_trips: self.allow_warehouse_round_trips, } } diff --git a/crates/sqlbuild-python-project/src/planner_identity/_helpers/execution.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/execution.rs new file mode 100644 index 0000000000..37f7a0b02f --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/_helpers/execution.rs @@ -0,0 +1,29 @@ +//! Execution order over native keys, or the cycle that prevents one. + +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::planner_identity::main::topological_order::topological_order; +use sqlbuild_analysis::planner_identity::models::GraphKey; +use std::collections::HashMap; + +/// The execution order of `upstream`, or the cycle that prevents one. +pub(crate) fn ordered_keys( + upstream: &[(ObjectKey, Vec)], + origins: &[((ObjectKey, ObjectKey), String)], +) -> Result, String> { + let key = |(kind, name): &ObjectKey| GraphKey { + kind: kind.clone(), + name: name.clone(), + }; + let edges: Vec<_> = upstream + .iter() + .map(|(node, deps)| (key(node), deps.iter().map(key).collect())) + .collect(); + let origins: HashMap<(GraphKey, GraphKey), String> = origins + .iter() + .map(|((gated, read), origin)| ((key(gated), key(read)), origin.clone())) + .collect(); + Ok(topological_order(&edges, &origins)? + .into_iter() + .map(|key| (key.kind, key.name)) + .collect()) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/_helpers/mod.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/mod.rs index 35817cbc58..dc18d0fcaa 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/_helpers/mod.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/_helpers/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod execution; pub(crate) mod graph; pub(crate) mod hashing; diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/execution_cycle.rs b/crates/sqlbuild-python-project/src/planner_identity/main/execution_cycle.rs new file mode 100644 index 0000000000..6f41164bc6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/execution_cycle.rs @@ -0,0 +1,18 @@ +//! `execution_cycle(project)`: the execution-order cycle every plan refuses with. + +use crate::planner_identity::_helpers::execution::ordered_keys; +use crate::planner_identity::main::execution_deps_with::execution_deps_with; +use crate::planner_identity::main::execution_origins_with::execution_origins_with; +use crate::references::main::native_execution_gate_edges::native_execution_gate_edges; +use crate::references::types::ExecutionGates; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// The cycle every plan refuses with, since plans order the whole execution graph before selecting. +pub fn execution_cycle(project: &Bound<'_, PyAny>) -> PyResult> { + let gates: ExecutionGates = native_execution_gate_edges(project)?; + Ok(ordered_keys( + &execution_deps_with(project, &gates)?, + &execution_origins_with(project, &gates)?, + ) + .err()) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps.rs b/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps.rs index 4e7fb11f96..705af9af47 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps.rs @@ -1,54 +1,11 @@ //! Execution dependencies kept native until a public graph or plan asks for its dict view. -use crate::compiled_views::main::project::compiled_models::compiled_models; -use crate::compiled_views::main::project::functions::functions; -use crate::compiled_views::main::project::seeds::seeds; -use crate::compiled_views::main::project::sources::sources; -use crate::compiled_views::main::project::sql_tests::sql_tests; -use crate::graph::main::nodes::native_key::native_key; -use crate::planner_identity::_helpers::graph::TABLE_FN_TEST_MODE; +use crate::planner_identity::main::execution_deps_with::execution_deps_with; use crate::references::main::native_execution_gate_edges::native_execution_gate_edges; -use crate::references::models::{GateRow, HookRow}; -use crate::resource_views::main::resource_reads::graph_row::graph_row; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_analysis::planner_identity::main::execution_upstream::execution_upstream; -use sqlbuild_analysis::planner_identity::models::ExecutionTest; -use sqlbuild_python_base::records::main::fields::field::field as native_field; /// Execution dependencies kept native until a public graph or plan asks for its dict view. pub fn execution_deps(project: &Bound<'_, PyAny>) -> PyResult)>> { - let mut resources: Vec<(ObjectKey, Vec)> = Vec::new(); - let collections: [Bound<'_, PyAny>; 4] = [ - compiled_models(project)?, - sources(project)?, - seeds(project)?, - functions(project)?, - ]; - for collection in collections { - for resource in collection.try_iter()? { - let row = graph_row(&resource?)?; - resources.push((row.key, row.deps)); - } - } - let mut tests: Vec = Vec::new(); - for test in sql_tests(project)?.try_iter()? { - let test = test?; - tests.push(ExecutionTest { - key: native_key(&native_field(&test, "key")?)?, - scope_deps: native_field(&test, "scope_deps")? - .try_iter()? - .map(|key| native_key(&key?)) - .collect::>()?, - table_function: native_field(&test, "mode")?.str()?.extract::()? - == TABLE_FN_TEST_MODE, - }); - } - let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; - let injected: Vec<_> = gates - .into_iter() - .map(|gate| (gate.gated, gate.read)) - .chain(hooks.into_iter().map(|hook| (hook.gated, hook.read))) - .collect(); - Ok(execution_upstream(resources, &tests, &injected)) + execution_deps_with(project, &native_execution_gate_edges(project)?) } diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps_with.rs b/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps_with.rs new file mode 100644 index 0000000000..0af49b155e --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/execution_deps_with.rs @@ -0,0 +1,59 @@ +//! Execution dependencies over gate edges already computed for this project. + +use crate::compiled_views::main::project::compiled_models::compiled_models; +use crate::compiled_views::main::project::functions::functions; +use crate::compiled_views::main::project::seeds::seeds; +use crate::compiled_views::main::project::sources::sources; +use crate::compiled_views::main::project::sql_tests::sql_tests; +use crate::graph::main::nodes::native_key::native_key; +use crate::planner_identity::_helpers::graph::TABLE_FN_TEST_MODE; +use crate::references::types::ExecutionGates; +use crate::resource_views::main::resource_reads::graph_row::graph_row; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::planner_identity::main::execution_upstream::execution_upstream; +use sqlbuild_analysis::planner_identity::models::ExecutionTest; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Execution dependencies over gate edges already computed for this project. +pub fn execution_deps_with( + project: &Bound<'_, PyAny>, + (gates, hooks): &ExecutionGates, +) -> PyResult)>> { + let mut resources: Vec<(ObjectKey, Vec)> = Vec::new(); + let collections: [Bound<'_, PyAny>; 4] = [ + compiled_models(project)?, + sources(project)?, + seeds(project)?, + functions(project)?, + ]; + for collection in collections { + for resource in collection.try_iter()? { + let row = graph_row(&resource?)?; + resources.push((row.key, row.deps)); + } + } + let mut tests: Vec = Vec::new(); + for test in sql_tests(project)?.try_iter()? { + let test = test?; + tests.push(ExecutionTest { + key: native_key(&native_field(&test, "key")?)?, + scope_deps: native_field(&test, "scope_deps")? + .try_iter()? + .map(|key| native_key(&key?)) + .collect::>()?, + table_function: native_field(&test, "mode")?.str()?.extract::()? + == TABLE_FN_TEST_MODE, + }); + } + let injected: Vec<_> = gates + .iter() + .map(|gate| (gate.gated.clone(), gate.read.clone())) + .chain( + hooks + .iter() + .map(|hook| (hook.gated.clone(), hook.read.clone())), + ) + .collect(); + Ok(execution_upstream(resources, &tests, &injected)) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/execution_order.rs b/crates/sqlbuild-python-project/src/planner_identity/main/execution_order.rs index d0bb0282e4..9253344286 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/main/execution_order.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/main/execution_order.rs @@ -1,11 +1,9 @@ //! Stable native execution order, projecting no Python graph or key objects. +use crate::planner_identity::_helpers::execution::ordered_keys; use crate::planner_identity::_helpers::graph::planner_input_error; use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_analysis::planner_identity::main::topological_order::topological_order; -use sqlbuild_analysis::planner_identity::models::GraphKey; -use std::collections::HashMap; /// Stable native execution order, projecting no Python graph or key objects. pub fn execution_order( @@ -13,21 +11,5 @@ pub fn execution_order( upstream: &[(ObjectKey, Vec)], origins: &[((ObjectKey, ObjectKey), String)], ) -> PyResult> { - let key = |(kind, name): &ObjectKey| GraphKey { - kind: kind.clone(), - name: name.clone(), - }; - let edges: Vec<_> = upstream - .iter() - .map(|(node, deps)| (key(node), deps.iter().map(key).collect())) - .collect(); - let origins: HashMap<(GraphKey, GraphKey), String> = origins - .iter() - .map(|((gated, read), origin)| ((key(gated), key(read)), origin.clone())) - .collect(); - Ok(topological_order(&edges, &origins) - .map_err(|message| planner_input_error(py, message))? - .into_iter() - .map(|key| (key.kind, key.name)) - .collect()) + ordered_keys(upstream, origins).map_err(|message| planner_input_error(py, message)) } diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins.rs b/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins.rs index 75e0769e49..ba161cbbbb 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins.rs @@ -1,78 +1,13 @@ //! Injected-edge descriptions remain native for cycle and buildability diagnostics. -use crate::compiled_views::main::project::sql_tests::sql_tests; -use crate::graph::main::nodes::native_key::native_key; -use crate::graph::models::NativeProjectGraph; -use crate::planner_identity::_helpers::graph::hook_label; +use crate::planner_identity::main::execution_origins_with::execution_origins_with; use crate::references::main::native_execution_gate_edges::native_execution_gate_edges; -use crate::references::models::{GateRow, HookRow}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_analysis::graph::main::edge_keys::edge_keys; -use sqlbuild_analysis::graph::models::EdgeDirection; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use std::collections::HashSet; /// Injected-edge descriptions remain native for cycle and buildability diagnostics. pub fn execution_origins( project: &Bound<'_, PyAny>, ) -> PyResult> { - let mut origins: Vec<((ObjectKey, ObjectKey), String)> = Vec::new(); - for test in sql_tests(project)?.try_iter()? { - let test = test?; - let test_name: String = native_field(&test, "name")?.extract()?; - for target_key in native_field(&test, "scope_deps")?.try_iter()? { - let target_key = native_key(&target_key?)?; - let target_name = &target_key.1; - let description = format!("SQL test '{test_name}' runs before '{target_name}'"); - origins.push(( - (target_key, native_key(&native_field(&test, "key")?)?), - description, - )); - } - } - let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; - let lineage = native_field(project, "lineage_graph")?; - let lineage = lineage.cast::()?.borrow(); - let is_lineage = |gated: &ObjectKey, read: &ObjectKey| { - edge_keys(lineage.graph(), gated, EdgeDirection::Upstream).contains(read) - }; - let mut seen: HashSet<_> = origins.iter().map(|(edge, _)| edge.clone()).collect(); - for GateRow { - audit_name, - target, - gated, - read, - } in &gates - { - if is_lineage(gated, read) || !seen.insert((gated.clone(), read.clone())) { - continue; - } - origins.push(( - (gated.clone(), read.clone()), - format!("audit '{audit_name}' on '{}' reads '{}'", target.1, read.1), - )); - } - for HookRow { - hook_name, - gated, - read, - hook_type: kind, - .. - } in &hooks - { - if is_lineage(gated, read) || !seen.insert((gated.clone(), read.clone())) { - continue; - } - origins.push(( - (gated.clone(), read.clone()), - format!( - "{} on '{}' reads '{}'", - hook_label(hook_name, kind), - gated.1, - read.1 - ), - )); - } - Ok(origins) + execution_origins_with(project, &native_execution_gate_edges(project)?) } diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins_with.rs b/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins_with.rs new file mode 100644 index 0000000000..b5eb8136ef --- /dev/null +++ b/crates/sqlbuild-python-project/src/planner_identity/main/execution_origins_with.rs @@ -0,0 +1,78 @@ +//! Injected-edge descriptions over gate edges already computed for this project. + +use crate::compiled_views::main::project::sql_tests::sql_tests; +use crate::graph::main::nodes::native_key::native_key; +use crate::graph::models::NativeProjectGraph; +use crate::planner_identity::_helpers::graph::hook_label; +use crate::references::models::{GateRow, HookRow}; +use crate::references::types::ExecutionGates; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::main::edge_keys::edge_keys; +use sqlbuild_analysis::graph::models::EdgeDirection; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::HashSet; + +/// Injected-edge descriptions over gate edges already computed for this project. +pub fn execution_origins_with( + project: &Bound<'_, PyAny>, + (gates, hooks): &ExecutionGates, +) -> PyResult> { + let mut origins: Vec<((ObjectKey, ObjectKey), String)> = Vec::new(); + for test in sql_tests(project)?.try_iter()? { + let test = test?; + let test_name: String = native_field(&test, "name")?.extract()?; + for target_key in native_field(&test, "scope_deps")?.try_iter()? { + let target_key = native_key(&target_key?)?; + let target_name = &target_key.1; + let description = format!("SQL test '{test_name}' runs before '{target_name}'"); + origins.push(( + (target_key, native_key(&native_field(&test, "key")?)?), + description, + )); + } + } + let lineage = native_field(project, "lineage_graph")?; + let lineage = lineage.cast::()?.borrow(); + let is_lineage = |gated: &ObjectKey, read: &ObjectKey| { + edge_keys(lineage.graph(), gated, EdgeDirection::Upstream).contains(read) + }; + let mut seen: HashSet<_> = origins.iter().map(|(edge, _)| edge.clone()).collect(); + for GateRow { + audit_name, + target, + gated, + read, + } in gates + { + if is_lineage(gated, read) || !seen.insert((gated.clone(), read.clone())) { + continue; + } + origins.push(( + (gated.clone(), read.clone()), + format!("audit '{audit_name}' on '{}' reads '{}'", target.1, read.1), + )); + } + for HookRow { + hook_name, + gated, + read, + hook_type: kind, + .. + } in hooks + { + if is_lineage(gated, read) || !seen.insert((gated.clone(), read.clone())) { + continue; + } + origins.push(( + (gated.clone(), read.clone()), + format!( + "{} on '{}' reads '{}'", + hook_label(hook_name, kind), + gated.1, + read.1 + ), + )); + } + Ok(origins) +} diff --git a/crates/sqlbuild-python-project/src/planner_identity/main/mod.rs b/crates/sqlbuild-python-project/src/planner_identity/main/mod.rs index d007f0bb8b..57f37779be 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/main/mod.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/main/mod.rs @@ -1,8 +1,11 @@ pub mod baselined_function_definition_hash; pub mod baselined_model_query_hash; +pub mod execution_cycle; pub mod execution_deps; +pub mod execution_deps_with; pub mod execution_order; pub mod execution_origins; +pub mod execution_origins_with; pub mod function_definition_hash; pub mod graph_key; pub mod model_definition_hash; diff --git a/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs b/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs index 639297c7ae..67987a66fd 100644 --- a/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs +++ b/crates/sqlbuild-python-project/src/planner_scope/_helpers/scope.rs @@ -5,15 +5,17 @@ use crate::compiled_views::main::project::compiled_models::compiled_models; use crate::compiled_views::main::project::sources::sources; use crate::graph::models::NativeProjectGraph; use crate::native_host::models::ModelHandle; -use crate::planner_identity::main::execution_deps::execution_deps; +use crate::planner_identity::main::execution_deps_with::execution_deps_with; use crate::planner_identity::main::execution_order::execution_order; -use crate::planner_identity::main::execution_origins::execution_origins; +use crate::planner_identity::main::execution_origins_with::execution_origins_with; use crate::planner_scope::main::keys::keys; use crate::planner_scope::main::project_graph::project_graph; use crate::planner_scope::models::ScopeData; use crate::planner_selection::main::py_strings::py_strings; use crate::planner_selection::main::resolve_graph_keys::resolve_graph_keys; use crate::planner_selection::models::Carrier; +use crate::references::main::native_execution_gate_edges::native_execution_gate_edges; +use crate::references::types::ExecutionGates; use crate::resource_views::main::source_reads::source_entry::source_entry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; @@ -54,7 +56,8 @@ pub(crate) fn build_planner_scope<'py>( let select = default_tuple(&args.value("select")?); let exclude = default_tuple(&args.value("exclude")?); let auto_load = args.value("auto_load_sources")?.is_truthy()?; - let mut upstream = execution_deps(&project)?; + let gates: ExecutionGates = native_execution_gate_edges(&project)?; + let mut upstream = execution_deps_with(&project, &gates)?; let lineage: Bound<'py, NativeProjectGraph> = native_field(&project, "lineage_graph")?.cast_into()?; let lineage = lineage.borrow(); @@ -110,7 +113,7 @@ pub(crate) fn build_planner_scope<'py>( .filter(|key| key.0 == SOURCE && (!auto_load || named_sources.contains(key))) .cloned() .collect(); - let order = execution_order(py, &upstream, &execution_origins(&project)?)?; + let order = execution_order(py, &upstream, &execution_origins_with(&project, &gates)?)?; let selected: HashSet<_> = selected.into_iter().collect(); let mut models: std::collections::HashMap = std::collections::HashMap::new(); diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs index f510f466b7..b4babc2ea7 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs @@ -254,6 +254,13 @@ pub(crate) fn compile_declarations( "job group {name} lists a job twice" ))); } + let members: BTreeSet = members + .into_iter() + .filter(|member| jobs.contains_key(member)) + .collect(); + if members.is_empty() { + continue; + } let dependencies = group_dependencies((name, &input), &jobs, &members) .map_err(|error| PyValueError::new_err(error.to_string()))?; let selection: BTreeSet = members diff --git a/crates/sqlbuild-python-project/src/references/mod.rs b/crates/sqlbuild-python-project/src/references/mod.rs index f89201734f..b6fcb4d65c 100644 --- a/crates/sqlbuild-python-project/src/references/mod.rs +++ b/crates/sqlbuild-python-project/src/references/mod.rs @@ -3,3 +3,4 @@ pub(crate) mod _helpers; pub mod main; pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/references/types.rs b/crates/sqlbuild-python-project/src/references/types.rs new file mode 100644 index 0000000000..8a821576a6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/references/types.rs @@ -0,0 +1,6 @@ +//! Public type aliases of references. + +use crate::references::models::{GateRow, HookRow}; + +/// The audit-gate and hook-read edges a plan injects, which both execution readers need. +pub type ExecutionGates = (Vec, Vec); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index f4caf1a4a6..a95586b7aa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -469,6 +469,10 @@ pub(crate) fn execute_build_plan<'py>( native_field(pipeline, "custom_materializations")?, ), ("loader_functions", lifecycle.loader_functions(py)?), + ( + "source_load_gate", + DirectPythonLifecycle::source_load_gate(&preparation.lifecycle)?.into_any(), + ), ], )?; let initial_state: Bound<'py, PyAny> = call_with( @@ -485,7 +489,7 @@ pub(crate) fn execute_build_plan<'py>( let result: Bound<'py, PyAny> = call_with( &host_entry(py, ExecutorHost::RunBuildPipeline)?, &[ - ("plan", plan.clone()), + ("plan", lifecycle.ordered_plan(&plan)?), ("connection_config", connection_config.clone()), ("adapter", adapter.clone()), ("settings", native_field(&project, "settings")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index f5bdab0275..3a4c3a8872 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -1,12 +1,14 @@ //! Ingress Python before the build and read-side tasks and assets dispatched as SQL completes. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::Mutex; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyDict; +use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyclass, pymethods}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::python_plan::models::RoundTripLoader; use sqlbuild_core::build_command::main::render_python_node_rows::render_python_node_rows; use sqlbuild_core::build_command::models::PythonNodeRow; use sqlbuild_core::cli_text::models::CliStyle; @@ -31,10 +33,14 @@ use sqlbuild_python_compile::compile_driver::main::setup::build_python_relation_ use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_project::graph::main::python_nodes::build_python_sql_run_lifecycle::build_python_sql_run_lifecycle; +use sqlbuild_python_project::graph::main::python_nodes::native_run_lifecycle::native_run_lifecycle; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_python_project::planner_scope::main::build_intermediate_source_map::build_intermediate_source_map; const SUCCESS_STATUS: &str = "success"; +const SKIPPED_STATUS: &str = "skipped"; +const HARD_SKIP: &str = "hard"; +const SOFT_SKIP: &str = "soft"; /// The cursor window and output settings the Python lifecycle runs with. pub(crate) struct LifecycleInputs<'py> { @@ -95,6 +101,10 @@ pub(crate) struct DirectPythonLifecycle { ingress_python_results: Py, ingress_load_results: Py, ingress_loader_names: Py, + /// Each round-trip loader's selected upstream tasks and assets, `(kind, name)`, which must succeed. + round_trip_gates: HashMap>, + /// Each round-trip loader's source load key and the SQL keys it waits for. + round_trip_waits: Vec<(ObjectKey, Vec)>, } #[pymethods] @@ -216,6 +226,75 @@ impl DirectPythonLifecycle { self.ingress_load_results.bind(py).clone() } + /// The gate a build consults before each source load, for round-trip loaders' upstreams. + pub(crate) fn source_load_gate<'py>( + slf: &Bound<'py, Self>, + ) -> PyResult> { + Bound::new( + slf.py(), + SourceLoadGate { + lifecycle: slf.clone().unbind(), + }, + ) + } + + /// `plan` with each round-trip loader's source load waiting for the SQL its upstream reads. + pub(crate) fn ordered_plan<'py>( + &self, + plan: &Bound<'py, PyAny>, + ) -> PyResult> { + if self.round_trip_waits.is_empty() { + return Ok(plan.clone()); + } + let py: Python<'py> = plan.py(); + let upstream: Bound<'py, PyDict> = native_field(plan, "upstream_deps")? + .cast_into::()? + .copy()?; + let downstream: Bound<'py, PyDict> = native_field(plan, "downstream_deps")? + .cast_into::()? + .copy()?; + let mut planned: HashMap> = HashMap::new(); + for key in native_field(plan, "execution_order")?.try_iter()? { + let key: Bound<'py, PyAny> = key?; + planned.insert( + ( + native_field(&key, "resource_type")?.extract()?, + native_field(&key, "name")?.extract()?, + ), + key, + ); + } + for (source, waits) in &self.round_trip_waits { + let Some(source) = planned.get(source) else { + continue; + }; + let waits: Vec<&Bound<'py, PyAny>> = + waits.iter().filter_map(|wait| planned.get(wait)).collect(); + let mut deps: Vec> = match upstream.get_item(source)? { + Some(existing) => existing.try_iter()?.collect::>()?, + None => Vec::new(), + }; + for wait in waits { + if !deps.iter().any(|dep| dep.eq(wait).unwrap_or(false)) { + deps.push(wait.clone()); + } + let mut dependents: Vec> = match downstream.get_item(wait)? { + Some(existing) => existing.try_iter()?.collect::>()?, + None => Vec::new(), + }; + if !dependents.iter().any(|dep| dep.eq(source).unwrap_or(false)) { + dependents.push(source.clone()); + } + downstream.set_item(wait, tuple(py, &dependents)?)?; + } + upstream.set_item(source, tuple(py, &deps)?)?; + } + let changes: Bound<'py, PyDict> = PyDict::new(py); + changes.set_item("upstream_deps", upstream)?; + changes.set_item("downstream_deps", downstream)?; + replaced_keywords(plan, &changes) + } + /// Close the read-side connection and report Python nodes that never ran. pub(crate) fn finalize(&self, py: Python<'_>) -> PyResult<()> { let connection: Option> = self @@ -303,6 +382,7 @@ struct LifecyclePlan<'py> { graph: Bound<'py, PyAny>, plan: Bound<'py, PyAny>, source_map: Bound<'py, PyAny>, + round_trips: Vec, } fn lifecycle_plan<'py>( @@ -328,6 +408,8 @@ fn lifecycle_plan<'py>( ], )?; let plan: Bound<'py, PyAny> = build_python_sql_run_lifecycle(&selection, &graph.clone())?; + let round_trips: Vec = + native_run_lifecycle(&selection, &graph)?.round_trip_loaders; let source_type: Bound<'py, PyAny> = enum_member(py, RecordId::CompiledResourceType, "SOURCE")?; let mut keys: Vec> = Vec::new(); for name in native_field(&plan, "ingress_loader_names")?.try_iter()? { @@ -351,6 +433,7 @@ fn lifecycle_plan<'py>( graph, plan, source_map, + round_trips, }) } @@ -518,6 +601,7 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( } read_side_tracker = Some(tracker.unbind()); } + let (round_trip_gates, round_trip_waits) = round_trip_ordering(&lifecycle, &plan_output)?; Bound::new( py, DirectPythonLifecycle { @@ -532,6 +616,84 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( ingress_python_results: ingress_python.unbind(), ingress_load_results: ingress_loads.unbind(), ingress_loader_names: native_field(&lifecycle.plan, "ingress_loader_names")?.unbind(), + round_trip_gates, + round_trip_waits, }, ) } + +type RoundTripGates = HashMap>; +type RoundTripWaits = Vec<(ObjectKey, Vec)>; + +/// Each round-trip loader's gating upstreams, and its source load key with the SQL it waits for. +fn round_trip_ordering( + lifecycle: &LifecyclePlan<'_>, + plan_output: &Bound<'_, PyAny>, +) -> PyResult<(RoundTripGates, RoundTripWaits)> { + let nodes: Bound<'_, PyAny> = native_field(&lifecycle.graph, "nodes_by_name")?; + let mut gates: RoundTripGates = HashMap::new(); + let mut waits: RoundTripWaits = Vec::new(); + for loader in &lifecycle.round_trips { + let mut kinds: Vec = Vec::new(); + for name in &loader.gates { + let kind: String = native_field(&nodes.get_item(name)?, "kind")?.extract()?; + kinds.push((kind, name.clone())); + } + gates.insert(loader.name.clone(), kinds); + for entry in native_field(plan_output, "source_load_entries")?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + if native_field(&entry, "loader")?.extract::()? == loader.name { + let key: Bound<'_, PyAny> = native_field(&entry, "key")?; + waits.push(( + ( + native_field(&key, "resource_type")?.extract()?, + native_field(&key, "name")?.extract()?, + ), + loader.waits_for.clone(), + )); + } + } + } + Ok((gates, waits)) +} + +/// The build's source-load gate: why a round-trip loader's load is skipped, and how. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct SourceLoadGate { + lifecycle: Py, +} + +#[pymethods] +impl SourceLoadGate { + /// Why a round-trip load is skipped: `hard` after a failed or skipped upstream, `soft` when it never ran. + fn __call__( + &self, + py: Python<'_>, + source: Bound<'_, PyAny>, + ) -> PyResult> { + let loader: Option = native_field(&source, "loader")?.extract()?; + let lifecycle: &DirectPythonLifecycle = self.lifecycle.get(); + let Some(gates) = loader.and_then(|loader| lifecycle.round_trip_gates.get(&loader)) else { + return Ok(None); + }; + let mut statuses: HashMap = HashMap::new(); + for result in lifecycle.python_results(py)?.try_iter()? { + let result: Bound<'_, PyAny> = result?; + statuses.insert( + native_field(&result, "node_name")?.extract()?, + native_field(&result, "status")?.str()?.extract()?, + ); + } + for (kind, name) in gates { + let (outcome, mode): (&str, &'static str) = match statuses.get(name).map(String::as_str) + { + Some(SUCCESS_STATUS) => continue, + Some(SKIPPED_STATUS) => ("was skipped", HARD_SKIP), + Some(_) => ("failed", HARD_SKIP), + None => ("did not run; the SQL it reads was not rebuilt", SOFT_SKIP), + }; + return Ok(Some((format!("Upstream {kind} '{name}' {outcome}"), mode))); + } + Ok(None) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs index f44cf2dc69..f8d5922e53 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/change_selection/definition_snapshot.rs @@ -62,9 +62,10 @@ const PROJECT_CONFIG_ELSEWHERE: [&str; 11] = [ ]; const LOCAL_CONFIG_ELSEWHERE: [&str; 4] = ["settings", "setting_overrides", "vars", "scenario"]; /// Settings that tune how a run executes or validates, not what a node computes. -const RUNTIME_SETTINGS: [&str; 9] = [ +const RUNTIME_SETTINGS: [&str; 10] = [ "sql_analysis", "require_sql_analysis", + "allow_warehouse_round_trips", "concurrency", "microbatch_concurrency", "test_timeout", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index 7206c88587..06fad8c8d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -71,7 +71,7 @@ const GRAPH_VIEW_NAMES: [&str; 5] = [ "path_index", ]; /// Selector kinds that match a set of resources rather than a named anchor. -const SET_SELECTOR_KINDS: [&str; 3] = ["tag", "path", "job"]; +const SET_SELECTOR_KINDS: [&str; 4] = ["tag", "path", "job", "kind"]; /// The relation graph lineage reads: a cached index, or a compiled graph with its adapter. enum LineageSource<'py> { diff --git a/fensu.toml b/fensu.toml index 3b0780ec42..e19a6994c7 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1346,6 +1346,14 @@ rule = "FFR601" path = "src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py" reason = "The SQL Server adapter intentionally keeps overrideable backend behavior on one class." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs"] +reason = "One reads a column's name and type for schema-change detection, the other the kind and name of each SQL resource a Python node reads; only the two-field record loop is alike." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs", "crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs"] +reason = "Each resolves `kind:` in its own selector engine: SQL selectors pick resource keys from the project graph and refuse Python kinds, Python selectors pick node names from the Python node index and refuse SQL kinds. Only the filter-then-refuse-empty shape is alike." + [[dupes.allowlist]] paths = ["crates/sqlbuild-rules/src/custom_rule_cache/main/parse_custom_rule_cache.rs", "crates/sqlbuild-rules/src/rule_cache/main/parse_rule_cache.rs"] reason = "Each reads its own cache format: the SQL rule cache checks its version and the build identity and returns findings by path, the custom-rule cache checks only its version, because its identities already include the build, and returns rules by code. Only the parse-and-filter shape is alike." @@ -2609,7 +2617,7 @@ pending_budgets = [ "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs=4", "crates/sqlbuild-python-project/src/graph/main/nodes/model_python_hook_names.rs=1", "crates/sqlbuild-python-project/src/graph/main/nodes/sql_ref_key.rs=1", - "crates/sqlbuild-python-project/src/graph/main/python_nodes/build_python_sql_run_lifecycle.rs=2", + "crates/sqlbuild-python-project/src/graph/main/python_nodes/native_run_lifecycle.rs=2", "crates/sqlbuild-python-project/src/graph/models.rs=2", "crates/sqlbuild-python-project/src/lineage/_helpers/lineage_facts.rs=1", "crates/sqlbuild-python-project/src/lineage/main/facts_eq.rs=2", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 98b07f8260..1136c6544e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -517,6 +517,7 @@ delta_compile = true | `scenario_timeout` | none | Time limit for each scenario, or `"disabled"` | | `delta_compile` | `true` | After an edit to model SQL or SQL test files only, compile just the changed files against the previous compile. See [Compiling after an edit](../cli/compile.md#compiling-after-an-edit). | | `allow_missing_secrets` | `false` | Let `compile`, `rules run`, `format`, `dag`, `lineage` and `scope` run without provider secrets. `sqlbuild_local.toml` only. See [Secrets](python-nodes/providers.md#secrets). | +| `allow_warehouse_round_trips` | `false` | Allow a loader to depend on a task or asset that reads SQL, so data can leave the warehouse and come back through a source. Project file only. See [The SQL boundary](python-nodes/overview.md#warehouse-round-trips). | ### Unit test and scenario limits diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md index 74df44465d..c439564e0f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -275,9 +275,9 @@ Every loader function receives a `LoaderContext` as its first argument. It provi | `qualify_name(name)` | Return a fully-qualified relation name in the destination database/schema | | `skip(reason, mode=...)` | Skip this loader. `mode` is `"soft"` (default, skip only this loader) or `"hard"` (also block dependents) | | `result(payload=, metadata=, materialized=)` | Return a structured result for a self-managed loader | -| `result_of(node_fn)` | Read the latest persisted result of an upstream node (current or previous run) | +| `result_of(node_fn)` | Read the result of an upstream task, asset or loader: its result from this run, else its latest successful one; raises naming the upstream if it never succeeded | | `results_of(node_fn, limit=N)` | Read the last N successful results of an upstream node, newest first | -| `loader(loader_fn)` | Return a `LoaderRelationRef` for an upstream loader dependency | +| `loader(loader_fn)` | Return a `LoaderRelationRef` for an upstream loader dependency; a task or asset is refused with a pointer to `result_of` | | `source(source_name)` | Return a `LoaderRelationRef` for a project source by YAML name | ### LoaderRelationRef @@ -292,7 +292,7 @@ Returned by `ctx.loader()` and `ctx.source()`. Provides access to an upstream re ## Loader dependencies -Loaders can depend on other loaders using `depends_on`. Dependencies are executed first, and their destination relations are available via `ctx.loader()`: +Loaders can depend on other loaders, tasks and assets using `depends_on`. Dependencies are executed first. An upstream loader's destination relation is available via `ctx.loader()`: ```python from sqlbuild.loaders import loader @@ -317,10 +317,38 @@ def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]: ] ``` -Read upstream relations through `ctx.loader(...)` and `ctx.source(...)`, not by writing their table names in SQL. SQLBuild rejects literal SQL that names a project source, model, or seed with `P008`, and warns at run time when dynamic SQL does; loaders run before models and seeds and cannot read them. See [Hard-coded relation names](sql-references.md#hard-coded-relation-names). +Read upstream relations through `ctx.loader(...)` and `ctx.source(...)`, not by writing their table names in SQL. SQLBuild rejects literal SQL that names a project source, model, or seed with `P008`, and warns at run time when dynamic SQL does; loaders cannot read models and seeds. To use model data in a loader, read it in an upstream task and pass it on through the task's result. See [Hard-coded relation names](sql-references.md#hard-coded-relation-names). Dependencies form a DAG. SQLBuild schedules loaders in topological order and executes independent loaders concurrently when `--concurrency` is set. +### Depending on tasks and assets + +A loader reads an upstream task's or asset's result with `ctx.result_of(...)`; `ctx.loader(...)` +takes loaders only. A failed or skipped upstream task or asset skips the loader and everything +downstream of the source it fills. A loader selected without its upstream task or asset reads that +upstream's latest successful result, and fails naming the upstream if it never succeeded. + +```python +from sqlbuild.loaders import loader +from sqlbuild.tasks import task + +@task +def fetch_partner_scores(ctx): + """Download the scores a partner published.""" + return ctx.result(payload={"scores": [[1, 10], [2, 20]]}) + +@loader(depends_on=[fetch_partner_scores]) +def partner_scores(ctx): + """Scores the partner returned for each order.""" + scores = ctx.result_of(node_function=fetch_partner_scores).payload["scores"] + return [{"order_id": order_id, "score": score} for order_id, score in scores] +``` + +When the task or asset reads SQL, directly or through other Python nodes, the loader makes a +[warehouse round trip](overview.md#warehouse-round-trips), which the project +must allow with `allow_warehouse_round_trips`. The loader then loads after the SQL the task reads +and before the models that read its source, in the same `sqb build`. + Intermediate loaders (those referenced only via `depends_on`, with no managed source of the same name) are given synthetic source entries and write to `__loader__` tables by default. Only the terminal loader - the one whose name matches a managed source - populates that source; intermediate loaders feed it. Use the `destination` parameter on the decorator to override the intermediate relation: ```python @@ -341,7 +369,7 @@ The `@loader` decorator accepts optional parameters that can also be set in the | Parameter | Description | |-----------|-------------| -| `depends_on` | List of loader functions this loader depends on | +| `depends_on` | List of loader, task and asset functions this loader depends on. Depending on one that reads SQL is a [warehouse round trip](overview.md#warehouse-round-trips) | | `destination` | Override the destination relation name (can include schema or database) | | `write_strategy` | `table`, `append`, `delete_insert`, or `merge` | | `cursor_column` | Column used for incremental cursor tracking | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index bbac4abad1..3c3ae4c95a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -66,8 +66,8 @@ Code that extends SQLBuild itself keeps its own folder, such as `rules/`, `mater The most important rule: **SQL models never depend on Python nodes.** The dependency direction is strictly one way. - SQL models depend only on other SQL resources (models, sources, seeds, functions). -- Python nodes may depend on other Python nodes. -- The only way Python data reaches SQL is through a **loader populating a source**: `loader -> source -> model`. +- Python nodes may depend on other Python nodes; a loader may depend on any Python node, including tasks and assets. +- Python data reaches SQL only through a **loader filling a source**: `loader -> source -> model`. - Tasks, assets, and checks may **read** SQL models, sources, and seeds at runtime through typed references (see [SQL references](sql-references.md)), but reading a model does not make it a SQL dependency. This keeps the SQL graph fully analyzable and testable on its own, while letting Python participate around the edges. @@ -85,6 +85,45 @@ This keeps the SQL graph fully analyzable and testable on its own, while letting model (SQL) ----> model (SQL) ``` +### Warehouse round trips + +A loader that depends on a task or asset that reads SQL, directly or through other Python nodes, +makes a warehouse round trip: data leaves the warehouse and comes back through the loader. For +example, a task reads `orders`, sends them to a partner's scoring service, and a loader loads the +scores it returned into the `partner_scores` source that `scored_orders` reads: + +``` + model orders --> task score_orders --> loader partner_scores --> source partner_scores --> model scored_orders +``` + +Round trips are off by default. Turn them on with `allow_warehouse_round_trips` in +`sqlbuild_project.toml` (see [Settings](../project-configuration.md#settings)); without it, +a plan or build that includes such a loader stops with an error naming the loader and the task or +asset that reads SQL. A loader that depends on a task or asset that reads no SQL is ordinary +ingestion and is always allowed. + +```toml +[settings] +allow_warehouse_round_trips = true +``` + +With the setting on: + +- One `sqb build` runs the SQL the task reads, then the task, then the loader, then the models that + read the loader's source. If the task fails or is skipped, the loader and everything downstream of + its source are skipped. +- Lineage and column lineage restart at the source: the source's columns come from the loader, not + from the models the task read. +- Job groups run a member that builds the task before the member that runs the loader, and that one + before the members that build models reading its source. +- `--changed` follows the chain: when the SQL the task reads is rebuilt, the task and loader run and + the models reading the source rebuild. When it is not rebuilt, the task does not run and the + loader is skipped softly, so the source keeps its data and models selected for their own changes + still build. +- A task that reads anything built from the source its own downstream loader fills is a cycle, + which stops the plan or build with the path, such as + `loader 'partner_scores' -> task 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> loader 'partner_scores'`. + ## Decorators Every decorator accepts the same organizational metadata: @@ -121,7 +160,7 @@ Each node receives a context object as its first argument (`TaskContext`, `Asset | `ctx.query(sql)` / `ctx.execute_sql(sql)` | Run SQL on the connection | | `ctx.qualify_name(name)` | Qualify a relation name | | `ctx.relation(ref)` | Resolve a declared `model()`/`source()` reference to a relation | -| `ctx.result_of(node_fn)` | Read the latest persisted result of an upstream node (current or previous run) | +| `ctx.result_of(node_fn)` | Read the result of an upstream node: its result from this run, else its latest successful one | | `ctx.results_of(node_fn, limit=N)` | Read the last N successful results of an upstream node, newest first | | `ctx.providers` | Access discovered [providers](providers.md) by name | @@ -207,6 +246,8 @@ Python nodes run in two phases relative to SQL: - **Ingress (pre-SQL):** loaders, and tasks/assets that feed sources, run before SQL models are built. - **Read-side (post-SQL):** tasks/assets that read SQL run after their SQL dependencies are built. + A loader in a [warehouse round trip](#warehouse-round-trips) loads after them, and before the + models that read its source. Read-side tasks and assets follow the same selection rule as SQL models. A SQL dependency that is part of the run gates the node: it waits for that model, seed, or source load, and is skipped if the diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md index 9ba80be234..bef44c5e3f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md @@ -80,6 +80,21 @@ sqb build --select seed:waffle_types sqb build --select source:raw__orders ``` +### Kind + +Select every node of one kind: + +```bash +sqb build --select kind:seed +sqb build --select "kind:model,path:models/marts" +sqb build --select +kind:asset +``` + +The kinds are `model`, `source`, `seed`, `udf`, `table_fn`, `task`, `asset`, `loader` and +`check`, as the server's catalog names them. Kind selectors compose with `+`, `,` and `--exclude` +like tag selectors. An unknown kind is an error that names the closest kinds, and a kind your +project has no nodes of is an error, as an unused tag is. + ### Job Select the SQL and Python nodes a [job or job group](orchestration.md) builds. Job names are globally @@ -174,6 +189,11 @@ sqb build --select "tag:staging,path:models/finance" This selects only models that match *both* conditions - in this case, models tagged `staging` that are also under the `models/finance` directory. +Graph expansion applies to the part it is written on, before the parts are intersected. +`+orders,kind:source` selects `orders` and everything upstream of it, then keeps only the sources: +the sources `orders` reads. `kind:source,+orders` means the same. Spaces separate selectors, whose +results are unioned, so `tag:finance kind:seed` selects nodes tagged `finance` and every seed. + ## Combining select and exclude ```bash @@ -194,6 +214,7 @@ error[S007]: unknown selector name 'nonexistent_model' error[S007]: unknown selector name 'missing_*' error[S009]: no models found under path 'models/nonexistent'. error[S000]: no SQL resources or Python nodes found with tag 'nonexistent_tag' +error[S007]: unknown node kind 'modle' error[S003]: path selector 'fact_orders~' requires names on both sides of '~' error[S013]: selector 'order_status_rules' selects a unit test; only `sqb test` and `sqb build` accept unit-test selectors ``` diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 031d077c56..8d683a5772 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -12053,6 +12053,7 @@ class SettingsConfig(NativeRecord): scenario_timeout_seconds: int | None delta_compile: bool allow_missing_secrets: bool + allow_warehouse_round_trips: bool def __init__( self, sql_analysis: bool = ..., @@ -12073,6 +12074,7 @@ class SettingsConfig(NativeRecord): scenario_timeout_seconds: int | None = ..., delta_compile: bool = ..., allow_missing_secrets: bool = ..., + allow_warehouse_round_trips: bool = ..., ) -> None: ... @property def sql_validation(self) -> bool: ... @@ -13051,11 +13053,13 @@ class PythonSqlRunLifecyclePlan(NativeRecord): ingress_python_node_names: frozenset[str] ingress_loader_names: frozenset[str] read_side_python_node_names: frozenset[str] + round_trip_loader_names: frozenset[str] def __init__( self, ingress_python_node_names: frozenset[str], ingress_loader_names: frozenset[str], read_side_python_node_names: frozenset[str], + round_trip_loader_names: frozenset[str] = ..., ) -> None: ... class PythonIdentityDependency(NativeRecord): diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index e9b67b8169..84a73638cf 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -40,6 +40,7 @@ from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.auditing.types import AuditOutcome +from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.diagnostics.main.log_debug_event import log_debug_event @@ -213,6 +214,9 @@ def __init__( self._custom_materializations: Mapping[str, Callable[..., MaterializationResult]] = ( customizations.custom_materializations or {} ) + self._source_load_gate: Callable[[SourceEntry], tuple[str, str] | None] | None = ( + customizations.source_load_gate + ) self._loader_functions_by_name: dict[str, DiscoveredLoaderFunction] = { loader.name: loader for loader in customizations.loader_functions } @@ -423,6 +427,10 @@ def _run_serial(self) -> None: break key: CompiledObjectKey = self._ready.popleft() if key.resource_type == CompiledResourceType.SOURCE: + if key in self._blocked_keys: + self._record_skipped(key) + self._mark_complete(key) + continue if not self._pre_dispatch(key): self._mark_complete(key) continue @@ -474,6 +482,10 @@ def _run_concurrent(self) -> None: break key: CompiledObjectKey = self._ready.popleft() if key.resource_type == CompiledResourceType.SOURCE: + if key in self._blocked_keys: + self._record_skipped(key) + self._mark_complete(key) + continue if not self._pre_dispatch(key): self._mark_complete(key) continue @@ -953,6 +965,12 @@ def _execute_node_with_cost_context( def _execute_source_node( self, *, key: CompiledObjectKey, connection: Any ) -> LoadExecutionResult: + gate: Callable[[SourceEntry], tuple[str, str] | None] | None = self._source_load_gate + if gate is not None: + source: SourceEntry = self._plan.source_map[key.name] + skip: tuple[str, str] | None = gate(source) + if skip is not None: + return skipped_load_result(source=source, reason=skip[0], mode=SkipMode(skip[1])) return execute_build_source_node( key=key, plan=self._plan, @@ -1394,7 +1412,9 @@ def _handle_completion( self._load_results.append(result) if self._on_node_complete is not None: self._on_node_complete(result) - failed = result.status == ExecutionStatus.FAILED + failed = result.status == ExecutionStatus.FAILED or ( + result.status == ExecutionStatus.SKIPPED and result.skip_mode == SkipMode.HARD + ) elif isinstance(result, ModelExecutionResult): self._model_results.append(result) diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 8afc1983ef..5905b2702a 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -16,6 +16,7 @@ ModelPlanEntry, SeedPlanEntry, SnapshotsConfig, + SourceEntry, SourceLoadPlanEntry, SqlTestPlanEntry, TargetSettings, @@ -127,6 +128,8 @@ class BuildCustomizations: custom_materializations: Mapping[str, Callable[..., MaterializationResult]] | None = None loader_functions: tuple[DiscoveredLoaderFunction, ...] = () + source_load_gate: Callable[[SourceEntry], tuple[str, str] | None] | None = None + """Why and how (`hard` or `soft`) a source load is skipped before it starts.""" @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py index 7e236f68d5..7a0bed956b 100644 --- a/src/sqlbuild/executor/load/models.py +++ b/src/sqlbuild/executor/load/models.py @@ -17,7 +17,7 @@ from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) -from sqlbuild.compiler.python_nodes.types import SkipMode +from sqlbuild.compiler.python_nodes.types import PythonNodeKind, SkipMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.constants import LOADER_RELATION_QUALIFIER_SEPARATOR from sqlbuild.executor.load.types import LoadProgressCallback @@ -218,6 +218,13 @@ def loader(self, loader_fn: Callable[..., object]) -> LoaderRelationRef: relation_ref: LoaderRelationRef | None = self.loader_refs.get(loader_fn) if relation_ref is None: + identity: tuple[str, str] | None = self.result_node_identity(loader_fn) + if identity is not None and identity[0] != PythonNodeKind.LOADER.value: + kind, name = identity + raise ExecutorInputError( + f"ctx.loader() takes a loader, not the {kind} '{name}'; read its result " + f"with ctx.result_of(node_function={name}) instead" + ) raise ExecutorInputError("Unknown loader reference passed to ctx.loader(...)") self._record_resolved(relation_ref) return relation_ref diff --git a/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py b/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py index 7e5a36393f..3f21b2b6c6 100644 --- a/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py +++ b/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py @@ -50,6 +50,16 @@ def results_of( def result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: """Return the `(kind, name)` a dependency's results are stored under.""" + identity: tuple[str, str] | None = self.result_node_identity(node_function) + if identity is None: + raise ExecutorInputError( + "Python node result dependency must be a task, asset, or loader" + ) + return identity + + def result_node_identity(self, node_function: Callable[..., object]) -> tuple[str, str] | None: + """Return the `(kind, name)` of a decorated task, asset or loader, else None.""" + for attribute, kind in RESULT_DEPENDENCY_ATTRIBUTES: definition: object = getattr(node_function, attribute, None) if definition is None: @@ -58,4 +68,4 @@ def result_dependency_identity(self, node_function: Callable[..., object]) -> tu if not isinstance(name, str): raise ExecutorInputError("Python node result dependency has no resolved name") return kind, name - raise ExecutorInputError("Python node result dependency must be a task, asset, or loader") + return None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 7c6b8f8fc6..8728f33513 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1697,3 +1697,27 @@ class AdapterDefaultResultsFallbackE2ETestCase: kept_loader: str reading_task: str expected_payload_keys: tuple[str, str] + + +@dataclass(frozen=True) +class WarehouseRoundTripE2ETestCase: + """A build of orders -> score task -> partner loader -> source -> scored model.""" + + description: str + project_settings: str + task_body: str + expected_exit_code: int + expected_output_order: tuple[str, ...] + expected_output_fragments: tuple[str, ...] + expected_scored_rows: tuple[tuple[object, ...], ...] | None + expected_scored_table_exists: bool = True + + +@dataclass(frozen=True) +class ChangedRoundTripE2ETestCase: + """A second `--changed` build of a round trip after an optional change to `orders`.""" + + description: str + orders_sql: str + expected_output_fragments: tuple[str, ...] + expected_scored_rows: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index c6a0ec51e3..54df02477e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -18,6 +18,7 @@ ChangedBuildStep, ChangedWarehouse, DeferCloneBuildE2ETestCase, + WarehouseRoundTripE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( REPO_ROOT, @@ -1532,3 +1533,91 @@ def node_results_in(*, db_path: Path, schema: str, node_type: str) -> list[tuple f"WHERE node_type = '{node_type}' ORDER BY node_name" ), ) + + +ALLOW_ROUND_TRIPS: str = "\n[settings]\nallow_warehouse_round_trips = true\n" +SCORING_TASK: str = ( + ' relation = ctx.relation(model("orders"))\n' + ' rows = ctx.query(f"SELECT order_id FROM {relation} ORDER BY order_id").fetchall()\n' + ' return ctx.result(payload={"scores": [[row[0], row[0] * 10] for row in rows]})\n' +) +FAILING_SCORING_TASK: str = ' raise RuntimeError("partner service unavailable")\n' + + +def round_trip_project_files(*, project_settings: str, task_body: str) -> dict[str, str]: + """Orders -> score task -> partner loader -> partner_scores source -> scored_orders model.""" + + return { + "sqlbuild_project.toml": ( + 'name = "partner_scoring"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connections.local]\n" + 'database = "scoring.duckdb"\n\n' + "[targets.dev]\n" + 'connection = "local"\n' + 'schema = "main"\n' + project_settings + ), + "models/orders.sql": ( + 'MODEL (description "Orders to score", materialized table);\n\n' + "SELECT 1 AS order_id UNION ALL SELECT 2 AS order_id\n" + ), + "sources/partner.yml": ( + "sources:\n" + " - name: partner_scores\n" + " description: Scores the partner returned for each order\n" + " managed: true\n" + " write_strategy: table\n" + " columns:\n" + " - name: order_id\n" + " type: INTEGER\n" + " - name: score\n" + " type: INTEGER\n" + ), + "models/scored_orders.sql": ( + 'MODEL (description "Orders with their partner score", materialized table);\n\n' + "SELECT o.order_id, s.score\n" + 'FROM __ref("orders") AS o\n' + 'JOIN __source("partner_scores") AS s ON o.order_id = s.order_id\n' + ), + "python/tasks/scoring.py": ( + "from sqlbuild.refs import model\n" + "from sqlbuild.tasks import task\n\n\n" + '@task(depends_on=model("orders"))\n' + "def score_orders(ctx):\n" + ' """Send the orders to the partner and keep the scores it returns."""\n' + task_body + ), + "python/loaders/partner.py": ( + "from sqlbuild.loaders import loader\n\n" + "from python.tasks.scoring import score_orders\n\n\n" + "@loader(depends_on=[score_orders])\n" + "def partner_scores(ctx):\n" + ' """Load the scores the partner returned."""\n' + " result = ctx.result_of(node_function=score_orders)\n" + ' return [{"order_id": o, "score": s} for o, s in result.payload["scores"]]\n' + ), + } + + +def build_round_trip_project( + *, tmp_path: Path, test_case: WarehouseRoundTripE2ETestCase +) -> tuple[Path, str]: + """Build the round-trip project once, check its output, and return its database and output.""" + + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="partner_scoring", + repo_files=round_trip_project_files( + project_settings=test_case.project_settings, task_body=test_case.task_body + ), + ) + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + positions: list[int] = [output.find(line) for line in test_case.expected_output_order] + assert all(position >= 0 for position in positions), output + assert positions == sorted(positions), output + assert all(fragment in output for fragment in test_case.expected_output_fragments), output + return project_dir / "scoring.duckdb", output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index d7fd693423..594a573e9b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -2093,8 +2093,10 @@ def test_given_source_task_loader_chain_when_running_run_then_command_rejects_bo assert result.returncode == test_case.expected_exit_code combined_output: str = result.stdout + result.stderr - assert "Loader 'stage_orders' depends on Python node 'prepare_orders'" in combined_output - assert "depends on SQL" in combined_output + assert "loader 'stage_orders' depends on" in combined_output + assert "'prepare_orders'" in combined_output + assert "which reads SQL" in combined_output + assert "allow_warehouse_round_trips" in combined_output @pytest.mark.parametrize( @@ -2170,8 +2172,10 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou assert result.returncode == test_case.expected_exit_code combined_output: str = result.stdout + result.stderr - assert "Loader 'stage_orders' depends on Python node 'prepare_orders'" in combined_output - assert "depends on SQL" in combined_output + assert "loader 'stage_orders' depends on" in combined_output + assert "'prepare_orders'" in combined_output + assert "which reads SQL" in combined_output + assert "allow_warehouse_round_trips" in combined_output @pytest.mark.parametrize( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py new file mode 100644 index 0000000000..308596fae2 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py @@ -0,0 +1,176 @@ +"""E2E coverage for a warehouse round trip: SQL -> task -> loader -> source -> model in one build.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + ChangedRoundTripE2ETestCase, + WarehouseRoundTripE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + ALLOW_ROUND_TRIPS, + FAILING_SCORING_TASK, + SCORING_TASK, + build_round_trip_project, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + query_duckdb, + run_sqb, + table_exists, +) + + +@pytest.mark.parametrize( + "test_case", + [ + WarehouseRoundTripE2ETestCase( + description="one build runs the SQL, the task, the loader and the model in order", + project_settings=ALLOW_ROUND_TRIPS, + task_body=SCORING_TASK, + expected_exit_code=0, + expected_output_order=( + "table orders OK", + "task score_orders OK", + "source partner_scores OK", + "table scored_orders OK", + ), + expected_output_fragments=("rows=2",), + expected_scored_rows=((1, 10), (2, 20)), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_warehouse_round_trip_when_building_then_the_loader_runs_between_its_task_and_readers( + tmp_path: Path, + test_case: WarehouseRoundTripE2ETestCase, +) -> None: + database, output = build_round_trip_project(tmp_path=tmp_path, test_case=test_case) + + assert ( + tuple( + query_duckdb( + db_path=database, + sql="SELECT order_id, score FROM main.scored_orders ORDER BY order_id", + ) + ) + == test_case.expected_scored_rows + ), output + + +@pytest.mark.parametrize( + "test_case", + [ + WarehouseRoundTripE2ETestCase( + description="a failing task skips the loader and the models reading its source", + project_settings=ALLOW_ROUND_TRIPS, + task_body=FAILING_SCORING_TASK, + expected_exit_code=1, + expected_output_order=( + "table orders OK", + "task score_orders FAIL", + "source partner_scores SKIP", + "table scored_orders SKIP", + ), + expected_output_fragments=("hard skip: Upstream task 'score_orders' failed",), + expected_scored_rows=None, + expected_scored_table_exists=False, + ), + WarehouseRoundTripE2ETestCase( + description="without the setting the build refuses the round trip", + project_settings="", + task_body=SCORING_TASK, + expected_exit_code=1, + expected_output_order=(), + expected_output_fragments=( + "loader 'partner_scores' depends on task 'score_orders', which reads SQL", + "set `allow_warehouse_round_trips = true` under [settings]", + ), + expected_scored_rows=None, + expected_scored_table_exists=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_are_not_built( + tmp_path: Path, + test_case: WarehouseRoundTripE2ETestCase, +) -> None: + database, output = build_round_trip_project(tmp_path=tmp_path, test_case=test_case) + + assert ( + table_exists(db_path=database, table_name="scored_orders") + == test_case.expected_scored_table_exists + ), output + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedRoundTripE2ETestCase( + description="a changed upstream model carries through the task, loader and source", + orders_sql=( + 'MODEL (description "Orders to score", materialized table);\n\n' + "SELECT 1 AS order_id UNION ALL SELECT 2 UNION ALL SELECT 3\n" + ), + expected_output_fragments=( + "task score_orders OK", + "source partner_scores OK", + "table scored_orders OK", + ), + expected_scored_rows=((1, 10), (2, 20), (3, 30)), + ), + ChangedRoundTripE2ETestCase( + description="unchanged SQL leaves the task unrun and the loader softly skipped", + orders_sql=( + 'MODEL (description "Orders to score", materialized table);\n\n' + "SELECT 1 AS order_id UNION ALL SELECT 2 AS order_id\n" + ), + expected_output_fragments=( + "soft skip: Upstream task 'score_orders' did not run; the SQL it reads was not " + "rebuilt", + "Completed successfully", + ), + expected_scored_rows=((1, 10), (2, 20)), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_built_round_trip_when_building_changed_then_the_decision_follows_the_chain( + tmp_path: Path, + test_case: ChangedRoundTripE2ETestCase, +) -> None: + database, _ = build_round_trip_project( + tmp_path=tmp_path, + test_case=WarehouseRoundTripE2ETestCase( + description="first build", + project_settings=ALLOW_ROUND_TRIPS, + task_body=SCORING_TASK, + expected_exit_code=0, + expected_output_order=(), + expected_output_fragments=(), + expected_scored_rows=None, + ), + ) + project_dir: Path = database.parent + (project_dir / "models" / "orders.sql").write_text(test_case.orders_sql, encoding="utf-8") + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--changed"), project_dir=project_dir + ) + + output: str = result.stdout + result.stderr + assert result.returncode == 0, output + assert all(fragment in output for fragment in test_case.expected_output_fragments), output + assert ( + tuple( + query_duckdb( + db_path=database, + sql="SELECT order_id, score FROM main.scored_orders ORDER BY order_id", + ) + ) + == test_case.expected_scored_rows + ), output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 8548909338..9e61257383 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1450,3 +1450,14 @@ class JobGroupCyclesCase: description: str fixed_group_toml: str expected_messages: tuple[str, ...] + + +@dataclass(frozen=True) +class GraphBoundaryCompileTestCase: + """A round-trip project edited into a graph every plan refuses, and what compile reports.""" + + description: str + project_settings: str + edited_files: dict[str, str] + expected_exit_code: int + expected_output_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_sql_graph_boundaries.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_sql_graph_boundaries.py new file mode 100644 index 0000000000..c33810f739 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_sql_graph_boundaries.py @@ -0,0 +1,134 @@ +"""`sqb compile` refuses a graph every plan refuses, whether or not anything selects its nodes.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + ALLOW_ROUND_TRIPS, + SCORING_TASK, + round_trip_project_files, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + GraphBoundaryCompileTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + run_sqb, +) + +CYCLE_TASK: str = ( + "from sqlbuild.refs import model\n" + "from sqlbuild.tasks import task\n\n\n" + '@task(depends_on=model("scored_orders"))\n' + "def score_orders(ctx):\n" + ' """Send the scored orders to the partner."""\n' + " return ctx.result(payload={})\n" +) +UNREAD_TASK: str = ( + "from sqlbuild.tasks import task\n\n\n" + "@task()\n" + "def score_orders(ctx):\n" + ' """Fetch the partner scores."""\n' + " return ctx.result(payload={})\n" +) +TERMINAL_LOADER_READER: str = ( + "from sqlbuild.tasks import task\n\n" + "from python.loaders.partner import partner_scores\n\n\n" + "@task(depends_on=[partner_scores])\n" + "def report_scores(ctx):\n" + ' """Report the scores the partner returned."""\n' + " return ctx.result(payload={})\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + GraphBoundaryCompileTestCase( + description="an allowed round trip compiles", + project_settings=ALLOW_ROUND_TRIPS, + edited_files={}, + expected_exit_code=0, + expected_output_fragments=(), + ), + GraphBoundaryCompileTestCase( + description="a cycle through a task, models, a source and its loader fails compile", + project_settings=ALLOW_ROUND_TRIPS, + edited_files={"python/tasks/scoring.py": CYCLE_TASK}, + expected_exit_code=1, + expected_output_fragments=( + "S000", + "loader 'partner_scores' is in a dependency cycle: loader 'partner_scores' -> " + "task 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> " + "loader 'partner_scores'", + ), + ), + GraphBoundaryCompileTestCase( + description="a cycle of model refs alone fails compile with the plan's message", + project_settings=ALLOW_ROUND_TRIPS, + edited_files={ + "models/order_totals.sql": ( + 'MODEL (description "Totals per order", materialized table);\n\n' + 'SELECT order_id FROM __ref("order_ranks")\n' + ), + "models/order_ranks.sql": ( + 'MODEL (description "Ranks per order", materialized table);\n\n' + 'SELECT order_id FROM __ref("order_totals")\n' + ), + }, + expected_exit_code=1, + expected_output_fragments=( + "error[S000]: Dependency cycle detected involving: " + "model:order_ranks, model:order_totals", + ), + ), + GraphBoundaryCompileTestCase( + description="a round trip without the setting fails compile", + project_settings="", + edited_files={}, + expected_exit_code=1, + expected_output_fragments=( + "loader 'partner_scores' depends on task 'score_orders', which reads SQL", + "set `allow_warehouse_round_trips = true` under [settings]", + ), + ), + GraphBoundaryCompileTestCase( + description="a task depending on a terminal loader instead of its source fails compile", + project_settings="", + edited_files={ + "python/tasks/scoring.py": UNREAD_TASK, + "python/tasks/reporting.py": TERMINAL_LOADER_READER, + }, + expected_exit_code=1, + expected_output_fragments=( + "Python node 'report_scores' depends on terminal loader 'partner_scores'; " + "depend on source 'partner_scores' instead", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_graph_every_plan_refuses_when_compiling_then_compile_fails_with_the_same_error( + tmp_path: Path, + test_case: GraphBoundaryCompileTestCase, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="partner_scoring", + repo_files=round_trip_project_files( + project_settings=test_case.project_settings, task_body=SCORING_TASK + ) + | test_case.edited_files, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + assert all(fragment in output for fragment in test_case.expected_output_fragments), output diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py b/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py index 36917cf737..45444992a3 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py @@ -23,7 +23,10 @@ def prepare_python_nodes_integration_project(root: Path) -> Path: - """Create a small project with Python task, asset, loader, and check nodes.""" + """Create a small project with Python task, asset, loader, and check nodes. + + Its loader depends on a task that reads SQL, so the project allows warehouse round trips. + """ project_dir: Path = root / "python_nodes_project" models_dir: Path = project_dir / "models" @@ -46,6 +49,9 @@ def prepare_python_nodes_integration_project(root: Path) -> Path: "", "[targets.dev]", 'schema = "main"', + "", + "[settings]", + "allow_warehouse_round_trips = true", ) ) + "\n", diff --git a/tests/goldens/compiler/failure/local-config-unknown-key.json b/tests/goldens/compiler/failure/local-config-unknown-key.json index 60b80858db..69ff368e19 100644 --- a/tests/goldens/compiler/failure/local-config-unknown-key.json +++ b/tests/goldens/compiler/failure/local-config-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_local.toml settings contains unknown key(s): sql_analysiss. Allowed keys: allow_missing_secrets, auto_load_sources, check_declared_columns, column_contract_mode, concurrency, default_audit_run_scope, default_audit_severity, delta_compile, max_concurrency, microbatch_concurrency, microbatch_unaccounted_partition_policy, query_change_tracking, require_sql_analysis, scenario_timeout, sql_analysis, sql_validation, table_promotion_mode, test_timeout, unit_test_max_real_models", + "message": "/sqlbuild_local.toml settings contains unknown key(s): sql_analysiss. Allowed keys: allow_missing_secrets, allow_warehouse_round_trips, auto_load_sources, check_declared_columns, column_contract_mode, concurrency, default_audit_run_scope, default_audit_severity, delta_compile, max_concurrency, microbatch_concurrency, microbatch_unaccounted_partition_policy, query_change_tracking, require_sql_analysis, scenario_timeout, sql_analysis, sql_validation, table_promotion_mode, test_timeout, unit_test_max_real_models", "help": "did you mean 'sql_analysis'?" } ] diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py index 9479f8f15c..0d7312db2f 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py @@ -565,7 +565,7 @@ def test_given_pending_postgres_expression_source_when_analysing_then_star_lists CyclicCompileTestCase( description="a ref cycle, a model reading itself and a star consumer of the cycle", files=_CYCLIC_PROJECT, - expected_exit_code=0, + expected_exit_code=1, expected_compiled=( "models/order_audit.sql", "models/order_summary.sql", diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index bad4b27454..d1649a493b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -117,7 +117,10 @@ def prepare_static_compile_project(root: Path) -> Path: def prepare_python_compile_project(root: Path) -> Path: - """Create a local project with Python nodes for compile DAG tests.""" + """Create a local project with Python nodes for compile DAG tests. + + Its loader depends on a task that reads SQL, so the project allows warehouse round trips. + """ project_dir: Path = prepare_python_dag_project(root) (project_dir / "sqlbuild_project.toml").write_text( @@ -129,6 +132,9 @@ def prepare_python_compile_project(root: Path) -> Path: "", "[targets.dev]", 'schema = "main"', + "", + "[settings]", + "allow_warehouse_round_trips = true", ) ) + "\n", diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py index 9f7fb862bd..cb1b756ef5 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py @@ -510,7 +510,7 @@ def test_given_invalid_sql_model_dependency_when_validating_boundaries_then_rais exclude=(), expected_error_type=ValueError, expected_error_fragment=( - "Loader 'load_events' depends on Python node 'prepare_orders' which depends on SQL" + "loader 'load_events' depends on task 'prepare_orders', which reads SQL" ), ) ], diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py index 573cdf857d..6baf9f4afa 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py @@ -42,6 +42,7 @@ def test_given_regular_primary_work_when_scheduler_submits_then_identity_is_prop scheduler: Any = object.__new__(BuildScheduler) scheduler._ready = deque((key,)) scheduler._in_flight = set() + scheduler._blocked_keys = set() scheduler._max_concurrency = 1 scheduler._stop = False scheduler._microbatch_coordinator_lock = threading.Lock() diff --git a/tests/unit/src/sqlbuild/executor/load/_test_types.py b/tests/unit/src/sqlbuild/executor/load/_test_types.py index 9e83615b8d..6512e8a92b 100644 --- a/tests/unit/src/sqlbuild/executor/load/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/load/_test_types.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass from datetime import date, datetime from typing import Any @@ -217,3 +218,12 @@ class LoaderOperationLifecycleTestCase: description: str expected_status: ExecutionStatus expected_event_types: tuple[str, ...] + + +@dataclass(frozen=True) +class LoaderReferenceRejectionTestCase: + """A function passed to `ctx.loader()` that is not a loader, and the error it raises.""" + + description: str + reference: Callable[..., object] + expected_message: str diff --git a/tests/unit/src/sqlbuild/executor/load/helpers.py b/tests/unit/src/sqlbuild/executor/load/helpers.py index c088800d27..1cd3a8ad89 100644 --- a/tests/unit/src/sqlbuild/executor/load/helpers.py +++ b/tests/unit/src/sqlbuild/executor/load/helpers.py @@ -5,7 +5,9 @@ from typing import Any, ClassVar from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.assets import asset from sqlbuild.observability import LifecycleEvent +from sqlbuild.tasks import task def operation_events(events: list[LifecycleEvent]) -> tuple[LifecycleEvent, ...]: @@ -75,3 +77,23 @@ def __init__(self) -> None: def connect(self, config: dict[str, object]) -> object: self.connection_count += 1 return super().connect(config) + + +@task +def score_orders(ctx: object) -> object: + """A task a loader may read through `ctx.result_of`, never `ctx.loader`.""" + + return ctx + + +@asset +def scored_orders_export(ctx: object) -> object: + """An asset a loader may read through `ctx.result_of`, never `ctx.loader`.""" + + return ctx + + +def plain_function(ctx: object) -> object: + """A function that is no Python node.""" + + return ctx diff --git a/tests/unit/src/sqlbuild/executor/load/test_models.py b/tests/unit/src/sqlbuild/executor/load/test_models.py index 6ae8590244..80174e76a2 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_models.py +++ b/tests/unit/src/sqlbuild/executor/load/test_models.py @@ -8,9 +8,18 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import LifeCycleEvent +from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.models import LoaderContext, LoaderRelationRef -from tests.unit.src.sqlbuild.executor.load._test_types import LoaderContextHelperTestCase -from tests.unit.src.sqlbuild.executor.load.helpers import LoaderContextTestAdapter +from tests.unit.src.sqlbuild.executor.load._test_types import ( + LoaderContextHelperTestCase, + LoaderReferenceRejectionTestCase, +) +from tests.unit.src.sqlbuild.executor.load.helpers import ( + LoaderContextTestAdapter, + plain_function, + score_orders, + scored_orders_export, +) @pytest.mark.parametrize( @@ -154,3 +163,57 @@ def fetch_events(ctx: object) -> object: assert context.source("raw_events").max("loaded_at") == test_case.expected_query_result events: tuple[LifeCycleEvent, ...] = statement_recorder.snapshot() assert tuple(event.content for event in events) == test_case.expected_recorded_events + + +@pytest.mark.parametrize( + "test_case", + [ + LoaderReferenceRejectionTestCase( + description="a task points to ctx.result_of", + reference=score_orders, + expected_message=( + "ctx.loader() takes a loader, not the task 'score_orders'; read its result with " + "ctx.result_of(node_function=score_orders) instead" + ), + ), + LoaderReferenceRejectionTestCase( + description="an asset points to ctx.result_of", + reference=scored_orders_export, + expected_message=( + "ctx.loader() takes a loader, not the asset 'scored_orders_export'; read its " + "result with ctx.result_of(node_function=scored_orders_export) instead" + ), + ), + LoaderReferenceRejectionTestCase( + description="an undecorated function stays an unknown loader reference", + reference=plain_function, + expected_message="Unknown loader reference passed to ctx.loader(...)", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_non_loader_when_passed_to_ctx_loader_then_the_error_names_what_to_use( + test_case: LoaderReferenceRejectionTestCase, +) -> None: + context: LoaderContext = LoaderContext( + adapter=LoaderContextTestAdapter(), + connection_config={}, + connection=object(), + destination="target_db.target_schema.partner_scores", + destination_database="target_db", + destination_schema="target_schema", + destination_name="partner_scores", + run_id="test_run", + target="dev", + vars={}, + is_reload=False, + use_color=False, + current_cursor_value=None, + logger=logging.getLogger("sqlbuild.loader.partner_scores"), + statement_recorder=StatementRecorder(), + ) + + with pytest.raises(ExecutorInputError) as error_info: + context.loader(test_case.reference) + + assert str(error_info.value) == test_case.expected_message diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 6c556efbf0..ab19864ac9 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -492,6 +492,7 @@ delta_compile = true | `scenario_timeout` | none | Time limit for each scenario, or `"disabled"` | | `delta_compile` | `true` | After an edit to model SQL or SQL test files only, compile just the changed files against the previous compile. See [Compiling after an edit](/docs/cli/compile#compiling-after-an-edit). | | `allow_missing_secrets` | `false` | Let `compile`, `rules run`, `format`, `dag`, `lineage` and `scope` run without provider secrets. `sqlbuild_local.toml` only. See [Secrets](/docs/concepts/python-nodes/providers#secrets). | +| `allow_warehouse_round_trips` | `false` | Allow a loader to depend on a task or asset that reads SQL, so data can leave the warehouse and come back through a source. Project file only. See [The SQL boundary](/docs/concepts/python-nodes/overview#warehouse-round-trips). | ### Unit test and scenario limits diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx index df12388f1b..5d0a06bbc3 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx @@ -255,9 +255,9 @@ Every loader function receives a `LoaderContext` as its first argument. It provi | `qualify_name(name)` | Return a fully-qualified relation name in the destination database/schema | | `skip(reason, mode=...)` | Skip this loader. `mode` is `"soft"` (default, skip only this loader) or `"hard"` (also block dependents) | | `result(payload=, metadata=, materialized=)` | Return a structured result for a self-managed loader | -| `result_of(node_fn)` | Read the latest persisted result of an upstream node (current or previous run) | +| `result_of(node_fn)` | Read the result of an upstream task, asset or loader: its result from this run, else its latest successful one; raises naming the upstream if it never succeeded | | `results_of(node_fn, limit=N)` | Read the last N successful results of an upstream node, newest first | -| `loader(loader_fn)` | Return a `LoaderRelationRef` for an upstream loader dependency | +| `loader(loader_fn)` | Return a `LoaderRelationRef` for an upstream loader dependency; a task or asset is refused with a pointer to `result_of` | | `source(source_name)` | Return a `LoaderRelationRef` for a project source by YAML name | ### LoaderRelationRef @@ -272,7 +272,7 @@ Returned by `ctx.loader()` and `ctx.source()`. Provides access to an upstream re ## Loader dependencies -Loaders can depend on other loaders using `depends_on`. Dependencies are executed first, and their destination relations are available via `ctx.loader()`: +Loaders can depend on other loaders, tasks and assets using `depends_on`. Dependencies are executed first. An upstream loader's destination relation is available via `ctx.loader()`: ```python from sqlbuild.loaders import loader @@ -297,10 +297,38 @@ def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]: ] ``` -Read upstream relations through `ctx.loader(...)` and `ctx.source(...)`, not by writing their table names in SQL. SQLBuild rejects literal SQL that names a project source, model, or seed with `P008`, and warns at run time when dynamic SQL does; loaders run before models and seeds and cannot read them. See [Hard-coded relation names](/docs/concepts/python-nodes/sql-references#hard-coded-relation-names). +Read upstream relations through `ctx.loader(...)` and `ctx.source(...)`, not by writing their table names in SQL. SQLBuild rejects literal SQL that names a project source, model, or seed with `P008`, and warns at run time when dynamic SQL does; loaders cannot read models and seeds. To use model data in a loader, read it in an upstream task and pass it on through the task's result. See [Hard-coded relation names](/docs/concepts/python-nodes/sql-references#hard-coded-relation-names). Dependencies form a DAG. SQLBuild schedules loaders in topological order and executes independent loaders concurrently when `--concurrency` is set. +### Depending on tasks and assets + +A loader reads an upstream task's or asset's result with `ctx.result_of(...)`; `ctx.loader(...)` +takes loaders only. A failed or skipped upstream task or asset skips the loader and everything +downstream of the source it fills. A loader selected without its upstream task or asset reads that +upstream's latest successful result, and fails naming the upstream if it never succeeded. + +```python +from sqlbuild.loaders import loader +from sqlbuild.tasks import task + +@task +def fetch_partner_scores(ctx): + """Download the scores a partner published.""" + return ctx.result(payload={"scores": [[1, 10], [2, 20]]}) + +@loader(depends_on=[fetch_partner_scores]) +def partner_scores(ctx): + """Scores the partner returned for each order.""" + scores = ctx.result_of(node_function=fetch_partner_scores).payload["scores"] + return [{"order_id": order_id, "score": score} for order_id, score in scores] +``` + +When the task or asset reads SQL, directly or through other Python nodes, the loader makes a +[warehouse round trip](/docs/concepts/python-nodes/overview#warehouse-round-trips), which the project +must allow with `allow_warehouse_round_trips`. The loader then loads after the SQL the task reads +and before the models that read its source, in the same `sqb build`. + Intermediate loaders (those referenced only via `depends_on`, with no managed source of the same name) are given synthetic source entries and write to `__loader__` tables by default. Only the terminal loader - the one whose name matches a managed source - populates that source; intermediate loaders feed it. Use the `destination` parameter on the decorator to override the intermediate relation: ```python @@ -321,7 +349,7 @@ The `@loader` decorator accepts optional parameters that can also be set in the | Parameter | Description | |-----------|-------------| -| `depends_on` | List of loader functions this loader depends on | +| `depends_on` | List of loader, task and asset functions this loader depends on. Depending on one that reads SQL is a [warehouse round trip](/docs/concepts/python-nodes/overview#warehouse-round-trips) | | `destination` | Override the destination relation name (can include schema or database) | | `write_strategy` | `table`, `append`, `delete_insert`, or `merge` | | `cursor_column` | Column used for incremental cursor tracking | diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 5954224c00..ff8e0475bf 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -50,8 +50,8 @@ Code that extends SQLBuild itself keeps its own folder, such as `rules/`, `mater The most important rule: **SQL models never depend on Python nodes.** The dependency direction is strictly one way. - SQL models depend only on other SQL resources (models, sources, seeds, functions). -- Python nodes may depend on other Python nodes. -- The only way Python data reaches SQL is through a **loader populating a source**: `loader -> source -> model`. +- Python nodes may depend on other Python nodes; a loader may depend on any Python node, including tasks and assets. +- Python data reaches SQL only through a **loader filling a source**: `loader -> source -> model`. - Tasks, assets, and checks may **read** SQL models, sources, and seeds at runtime through typed references (see [SQL references](/docs/concepts/python-nodes/sql-references)), but reading a model does not make it a SQL dependency. This keeps the SQL graph fully analyzable and testable on its own, while letting Python participate around the edges. @@ -69,6 +69,45 @@ This keeps the SQL graph fully analyzable and testable on its own, while letting model (SQL) ----> model (SQL) ``` +### Warehouse round trips + +A loader that depends on a task or asset that reads SQL, directly or through other Python nodes, +makes a warehouse round trip: data leaves the warehouse and comes back through the loader. For +example, a task reads `orders`, sends them to a partner's scoring service, and a loader loads the +scores it returned into the `partner_scores` source that `scored_orders` reads: + +``` + model orders --> task score_orders --> loader partner_scores --> source partner_scores --> model scored_orders +``` + +Round trips are off by default. Turn them on with `allow_warehouse_round_trips` in +`sqlbuild_project.toml` (see [Settings](/docs/concepts/project-configuration#settings)); without it, +a plan or build that includes such a loader stops with an error naming the loader and the task or +asset that reads SQL. A loader that depends on a task or asset that reads no SQL is ordinary +ingestion and is always allowed. + +```toml +[settings] +allow_warehouse_round_trips = true +``` + +With the setting on: + +- One `sqb build` runs the SQL the task reads, then the task, then the loader, then the models that + read the loader's source. If the task fails or is skipped, the loader and everything downstream of + its source are skipped. +- Lineage and column lineage restart at the source: the source's columns come from the loader, not + from the models the task read. +- Job groups run a member that builds the task before the member that runs the loader, and that one + before the members that build models reading its source. +- `--changed` follows the chain: when the SQL the task reads is rebuilt, the task and loader run and + the models reading the source rebuild. When it is not rebuilt, the task does not run and the + loader is skipped softly, so the source keeps its data and models selected for their own changes + still build. +- A task that reads anything built from the source its own downstream loader fills is a cycle, + which stops the plan or build with the path, such as + `loader 'partner_scores' -> task 'score_orders' -> model 'scored_orders' -> source 'partner_scores' -> loader 'partner_scores'`. + ## Decorators Every decorator accepts the same organizational metadata: @@ -105,7 +144,7 @@ Each node receives a context object as its first argument (`TaskContext`, `Asset | `ctx.query(sql)` / `ctx.execute_sql(sql)` | Run SQL on the connection | | `ctx.qualify_name(name)` | Qualify a relation name | | `ctx.relation(ref)` | Resolve a declared `model()`/`source()` reference to a relation | -| `ctx.result_of(node_fn)` | Read the latest persisted result of an upstream node (current or previous run) | +| `ctx.result_of(node_fn)` | Read the result of an upstream node: its result from this run, else its latest successful one | | `ctx.results_of(node_fn, limit=N)` | Read the last N successful results of an upstream node, newest first | | `ctx.providers` | Access discovered [providers](/docs/concepts/python-nodes/providers) by name | @@ -191,6 +230,8 @@ Python nodes run in two phases relative to SQL: - **Ingress (pre-SQL):** loaders, and tasks/assets that feed sources, run before SQL models are built. - **Read-side (post-SQL):** tasks/assets that read SQL run after their SQL dependencies are built. + A loader in a [warehouse round trip](#warehouse-round-trips) loads after them, and before the + models that read its source. Read-side tasks and assets follow the same selection rule as SQL models. A SQL dependency that is part of the run gates the node: it waits for that model, seed, or source load, and is skipped if the diff --git a/website/src/content/docs/docs/concepts/selectors.mdx b/website/src/content/docs/docs/concepts/selectors.mdx index 2d7e7e2a1e..c4c8b91e92 100644 --- a/website/src/content/docs/docs/concepts/selectors.mdx +++ b/website/src/content/docs/docs/concepts/selectors.mdx @@ -67,6 +67,21 @@ sqb build --select seed:waffle_types sqb build --select source:raw__orders ``` +### Kind + +Select every node of one kind: + +```bash +sqb build --select kind:seed +sqb build --select "kind:model,path:models/marts" +sqb build --select +kind:asset +``` + +The kinds are `model`, `source`, `seed`, `udf`, `table_fn`, `task`, `asset`, `loader` and +`check`, as the server's catalog names them. Kind selectors compose with `+`, `,` and `--exclude` +like tag selectors. An unknown kind is an error that names the closest kinds, and a kind your +project has no nodes of is an error, as an unused tag is. + ### Job Select the SQL and Python nodes a [job or job group](/docs/concepts/orchestration) builds. Job names are globally @@ -161,6 +176,11 @@ sqb build --select "tag:staging,path:models/finance" This selects only models that match *both* conditions - in this case, models tagged `staging` that are also under the `models/finance` directory. +Graph expansion applies to the part it is written on, before the parts are intersected. +`+orders,kind:source` selects `orders` and everything upstream of it, then keeps only the sources: +the sources `orders` reads. `kind:source,+orders` means the same. Spaces separate selectors, whose +results are unioned, so `tag:finance kind:seed` selects nodes tagged `finance` and every seed. + ## Combining select and exclude ```bash @@ -181,6 +201,7 @@ error[S007]: unknown selector name 'nonexistent_model' error[S007]: unknown selector name 'missing_*' error[S009]: no models found under path 'models/nonexistent'. error[S000]: no SQL resources or Python nodes found with tag 'nonexistent_tag' +error[S007]: unknown node kind 'modle' error[S003]: path selector 'fact_orders~' requires names on both sides of '~' error[S013]: selector 'order_status_rules' selects a unit test; only `sqb test` and `sqb build` accept unit-test selectors ```